Skip to content

Commit 7104d64

Browse files
Yuhong Guowesm
authored andcommitted
ARROW-3187: [C++] Add support for using glog (Google logging library)
1. `glog` provides richer information. 2. `glog` can print good call stack while crashing, which is very helpful for debugging. 3. Make logging pluggable with `glog` or original log using a macro. Users can enable/disable `glog` using the cmake option `ARROW_USE_GLOG`. Author: Yuhong Guo <yuhong.gyh@antfin.com> Author: Wes McKinney <wesm+git@apache.org> Closes apache#2522 from guoyuhong/glog and squashes the following commits: b359640 <Yuhong Guo> Revert some useless changes. 38560c0 <Yuhong Guo> Change back the test code to fix logging-test e3203a5 <Wes McKinney> Some fixes, run logging-test 4a9d172 <Wes McKinney> Fix Flatbuffers download url f364308 <Yuhong Guo> Add test code to only include glog lib and init it without other use. c8269fd <Yuhong Guo> Change ARROW_JEMALLOC_LINK_LIBS setting to ARROW_LINK_LIBS 34e6841 <Yuhong Guo> Add pthread 48afa34 <Yuhong Guo> Address comment 12f9ba7 <Yuhong Guo> Disable glog from ARROW_BUILD_TOOLCHAIN 62f2000 <Yuhong Guo> Add -pthread to glog 673dbeb <Yuhong Guo> Try to fix ci FAILURE 69c1e79 <Yuhong Guo> Add pthread for glog fbe9cc9 <Yuhong Guo> Change Thirdpart to use EP_CXX_FLAGS 6f4d1b8 <Yuhong Guo> Add lib64 to lib path suffix. 84532e3 <Yuhong Guo> Add glog to Dockerfile ccc03cb <Yuhong Guo> Fix a bug 7bacd53 <Yuhong Guo> Add LICENSE information. 9a3834c <Yuhong Guo> Enable glog and fix building error 2b1f7e0 <Yuhong Guo> Turn glog off. 7d92091 <Yuhong Guo> Hide glog symbols from libarrow.so a6ff671 <Yuhong Guo> Support offline build of glog 14865ee <Yuhong Guo> Try to fix MSVC building failure 53ceceb <Yuhong Guo> Change log level to enum and refine code 09c6af7 <Yuhong Guo> Enable glog in plasma
1 parent 92a2e6a commit 7104d64

18 files changed

Lines changed: 651 additions & 93 deletions

LICENSE.txt

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,3 +696,23 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
696696
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
697697
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
698698
SOFTWARE.
699+
700+
--------------------------------------------------------------------------------
701+
702+
cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and
703+
cpp/src/arrow/util/logging-test.cc are adapted from
704+
Ray Project (https://github.com/ray-project/ray) (Apache 2.0).
705+
706+
Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray)
707+
708+
Licensed under the Apache License, Version 2.0 (the "License");
709+
you may not use this file except in compliance with the License.
710+
You may obtain a copy of the License at
711+
712+
http://www.apache.org/licenses/LICENSE-2.0
713+
714+
Unless required by applicable law or agreed to in writing, software
715+
distributed under the License is distributed on an "AS IS" BASIS,
716+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
717+
See the License for the specific language governing permissions and
718+
limitations under the License.

ci/travis_install_toolchain.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,6 @@ if [ ! -e $CPP_TOOLCHAIN ]; then
3939
snappy \
4040
thrift-cpp=0.11.0 \
4141
zlib \
42+
glog \
4243
zstd
4344
fi

cpp/CMakeLists.txt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,10 @@ Pass multiple labels by dividing with semicolons")
256256
"If off, 'quiet' flags will be passed to linting tools"
257257
OFF)
258258

259+
option(ARROW_USE_GLOG
260+
"Build libraries with glog support for pluggable logging"
261+
ON)
262+
259263
if (MSVC)
260264
option(ARROW_USE_CLCACHE
261265
"Use clcache if available"
@@ -302,6 +306,8 @@ endif()
302306
if (MSVC)
303307
# ORC doesn't build on windows
304308
set(ARROW_ORC OFF)
309+
# Plasma using glog is not fully tested on windows.
310+
set(ARROW_USE_GLOG OFF)
305311
endif()
306312

307313
if(ARROW_ORC)
@@ -642,6 +648,11 @@ if (ARROW_ORC)
642648
orc)
643649
endif()
644650
651+
if (ARROW_USE_GLOG)
652+
#add_definitions(-DARROW_USE_GLOG)
653+
SET(ARROW_STATIC_LINK_LIBS glog_static ${ARROW_STATIC_LINK_LIBS})
654+
endif()
655+
645656
if (ARROW_STATIC_LINK_LIBS)
646657
add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
647658
endif()

cpp/cmake_modules/FindGLOG.cmake

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
# Tries to find GLog headers and libraries.
15+
#
16+
# Usage of this module as follows:
17+
#
18+
# find_package(GLOG)
19+
#
20+
# Variables used by this module, they can change the default behaviour and need
21+
# to be set before calling find_package:
22+
#
23+
# GLOG_HOME - When set, this path is inspected instead of standard library
24+
# locations as the root of the GLog installation.
25+
# The environment variable GLOG_HOME overrides this veriable.
26+
#
27+
# This module defines
28+
# GLOG_INCLUDE_DIR, directory containing headers
29+
# GLOG_STATIC_LIB, path to libglog.a
30+
# GLOG_FOUND, whether glog has been found
31+
32+
if( NOT "${GLOG_HOME}" STREQUAL "")
33+
file( TO_CMAKE_PATH "${GLOG_HOME}" _native_path )
34+
list( APPEND _glog_roots ${_native_path} )
35+
endif()
36+
37+
message(STATUS "GLOG_HOME: ${GLOG_HOME}")
38+
# Try the parameterized roots, if they exist
39+
if ( _glog_roots )
40+
set (lib_dirs "lib")
41+
if (EXISTS "${_glog_roots}/lib64")
42+
set (lib_dirs "lib64" ${lib_dirs})
43+
endif ()
44+
45+
find_path( GLOG_INCLUDE_DIR NAMES glog/logging.h
46+
PATHS ${_glog_roots} NO_DEFAULT_PATH
47+
PATH_SUFFIXES "include" )
48+
find_library( GLOG_LIBRARIES NAMES glog
49+
PATHS ${_glog_roots} NO_DEFAULT_PATH
50+
PATH_SUFFIXES ${lib_dirs})
51+
else ()
52+
find_path( GLOG_INCLUDE_DIR NAMES glog/logging.h )
53+
find_library( GLOG_LIBRARIES NAMES glog )
54+
endif ()
55+
56+
57+
if (GLOG_INCLUDE_DIR AND GLOG_LIBRARIES)
58+
set(GLOG_FOUND TRUE)
59+
get_filename_component( GLOG_LIBS ${GLOG_LIBRARIES} PATH )
60+
set(GLOG_LIB_NAME glog)
61+
set(GLOG_STATIC_LIB ${GLOG_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GLOG_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
62+
set(GLOG_SHARED_LIB ${GLOG_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GLOG_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
63+
else ()
64+
set(GLOG_FOUND FALSE)
65+
endif ()
66+
67+
if (GLOG_FOUND)
68+
if (NOT GLOG_FIND_QUIETLY)
69+
message(STATUS "Found the GLog library: ${GLOG_LIBRARIES}")
70+
endif ()
71+
else ()
72+
if (NOT GLOG_FIND_QUIETLY)
73+
set(GLOG_ERR_MSG "Could not find the GLog library. Looked in ")
74+
if ( _glog_roots )
75+
set(GLOG_ERR_MSG "${GLOG_ERR_MSG} ${_glog_roots}.")
76+
else ()
77+
set(GLOG_ERR_MSG "${GLOG_ERR_MSG} system search paths.")
78+
endif ()
79+
if (GLOG_FIND_REQUIRED)
80+
message(FATAL_ERROR "${GLOG_ERR_MSG}")
81+
else (GLOG_FIND_REQUIRED)
82+
message(STATUS "${GLOG_ERR_MSG}")
83+
endif (GLOG_FIND_REQUIRED)
84+
endif ()
85+
endif ()
86+
87+
mark_as_advanced(
88+
GLOG_INCLUDE_DIR
89+
GLOG_LIBS
90+
GLOG_LIBRARIES
91+
GLOG_STATIC_LIB
92+
)

cpp/cmake_modules/ThirdpartyToolchain.cmake

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,14 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
3333
set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3434
set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3535
# orc disabled as it's not in conda-forge (but in Anaconda with an incompatible ABI)
36-
# set(ORC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
36+
# set(ORC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3737
set(PROTOBUF_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3838
set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3939
set(SNAPPY_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
4040
set(THRIFT_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
4141
set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
4242
set(ZSTD_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
43+
set(GLOG_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
4344

4445
if (NOT DEFINED ENV{BOOST_ROOT})
4546
# Since we have to set this in the environment, we check whether
@@ -106,6 +107,10 @@ if (DEFINED ENV{ZSTD_HOME})
106107
set(ZSTD_HOME "$ENV{ZSTD_HOME}")
107108
endif()
108109

110+
if (DEFINED ENV{GLOG_HOME})
111+
set(GLOG_HOME "$ENV{GLOG_HOME}")
112+
endif()
113+
109114
# ----------------------------------------------------------------------
110115
# Some EP's require other EP's
111116

@@ -238,6 +243,12 @@ else()
238243
set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz")
239244
endif()
240245

246+
if (DEFINED ENV{ARROW_GLOG_URL})
247+
set(GLOG_SOURCE_URL "$ENV{ARROW_GLOG_URL}")
248+
else()
249+
set(GLOG_SOURCE_URL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz")
250+
endif()
251+
241252
# ----------------------------------------------------------------------
242253
# ExternalProject options
243254

@@ -1261,3 +1272,59 @@ if (THRIFT_VENDORED)
12611272
endif()
12621273

12631274
endif() # ARROW_HIVESERVER2
1275+
1276+
if (ARROW_USE_GLOG)
1277+
# ----------------------------------------------------------------------
1278+
# GLOG
1279+
1280+
if("${GLOG_HOME}" STREQUAL "")
1281+
set(GLOG_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-prefix/src/glog_ep")
1282+
set(GLOG_INCLUDE_DIR "${GLOG_BUILD_DIR}/include")
1283+
set(GLOG_STATIC_LIB "${GLOG_BUILD_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}glog${CMAKE_STATIC_LIBRARY_SUFFIX}")
1284+
set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
1285+
set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC")
1286+
if (PTHREAD_LIBRARY)
1287+
set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread")
1288+
set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC -pthread")
1289+
endif()
1290+
message(STATUS "GLOG_CMAKE_CXX_FLAGS: ${GLOG_CMAKE_CXX_FLAGS}")
1291+
message(STATUS "CMAKE_CXX_FLAGS in glog: ${GLOG_CMAKE_CXX_FLAGS}")
1292+
1293+
if(APPLE)
1294+
# If we don't set this flag, the binary built with 10.13 cannot be used in 10.12.
1295+
set(GLOG_CMAKE_CXX_FLAGS "${GLOG_CMAKE_CXX_FLAGS} -mmacosx-version-min=10.9")
1296+
endif()
1297+
1298+
set(GLOG_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
1299+
-DCMAKE_INSTALL_PREFIX=${GLOG_BUILD_DIR}
1300+
-DBUILD_SHARED_LIBS=OFF
1301+
-DBUILD_TESTING=OFF
1302+
-DWITH_GFLAGS=OFF
1303+
-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_CXX_FLAGS}
1304+
-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_C_FLAGS}
1305+
-DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS})
1306+
message(STATUS "Glog version: ${GLOG_VERSION}")
1307+
ExternalProject_Add(glog_ep
1308+
URL ${GLOG_SOURCE_URL}
1309+
BUILD_IN_SOURCE 1
1310+
BUILD_BYPRODUCTS "${GLOG_STATIC_LIB}"
1311+
CMAKE_ARGS ${GLOG_CMAKE_ARGS}
1312+
${EP_LOG_OPTIONS})
1313+
1314+
set(GLOG_VENDORED 1)
1315+
else()
1316+
find_package(GLOG REQUIRED)
1317+
set(GLOG_VENDORED 0)
1318+
endif()
1319+
1320+
message(STATUS "Glog include dir: ${GLOG_INCLUDE_DIR}")
1321+
message(STATUS "Glog static library: ${GLOG_STATIC_LIB}")
1322+
1323+
include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
1324+
ADD_THIRDPARTY_LIB(glog_static
1325+
STATIC_LIB ${GLOG_STATIC_LIB})
1326+
1327+
if (GLOG_VENDORED)
1328+
add_dependencies(glog_static glog_ep)
1329+
endif()
1330+
endif()

cpp/src/arrow/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ set(ARROW_SRCS
4444
util/io-util.cc
4545
util/key_value_metadata.cc
4646
util/thread-pool.cc
47+
util/logging.cc
4748
)
4849

4950
if ("${COMPILER_FAMILY}" STREQUAL "clang")

cpp/src/arrow/symbols.map

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@
7373
# Statically linked C++ dependencies
7474
boost::*;
7575
google::*;
76+
# glog
77+
*MakeCheckOpValueString*;
7678
orc::*;
7779
snappy::*;
7880
};

cpp/src/arrow/util/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ ADD_ARROW_TEST(parsing-util-test)
6262
ADD_ARROW_TEST(stl-util-test)
6363
ADD_ARROW_TEST(thread-pool-test)
6464
ADD_ARROW_TEST(lazy-test)
65+
ADD_ARROW_TEST(logging-test)
6566

6667
ADD_ARROW_BENCHMARK(bit-util-benchmark)
6768
ADD_ARROW_BENCHMARK(decimal-benchmark)

cpp/src/arrow/util/logging-test.cc

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <chrono>
19+
#include <cstdlib>
20+
#include <iostream>
21+
22+
#include "arrow/util/logging.h"
23+
#include "gtest/gtest.h"
24+
25+
// This code is adapted from
26+
// https://github.com/ray-project/ray/blob/master/src/ray/util/logging_test.cc.
27+
28+
namespace arrow {
29+
30+
int64_t current_time_ms() {
31+
std::chrono::milliseconds ms_since_epoch =
32+
std::chrono::duration_cast<std::chrono::milliseconds>(
33+
std::chrono::steady_clock::now().time_since_epoch());
34+
return ms_since_epoch.count();
35+
}
36+
37+
// This is not really test.
38+
// This file just print some information using the logging macro.
39+
40+
void PrintLog() {
41+
ARROW_LOG(DEBUG) << "This is the"
42+
<< " DEBUG"
43+
<< " message";
44+
ARROW_LOG(INFO) << "This is the"
45+
<< " INFO message";
46+
ARROW_LOG(WARNING) << "This is the"
47+
<< " WARNING message";
48+
ARROW_LOG(ERROR) << "This is the"
49+
<< " ERROR message";
50+
ARROW_CHECK(true) << "This is a ARROW_CHECK"
51+
<< " message but it won't show up";
52+
// The following 2 lines should not run since it will cause program failure.
53+
// ARROW_LOG(FATAL) << "This is the FATAL message";
54+
// ARROW_CHECK(false) << "This is a ARROW_CHECK message but it won't show up";
55+
}
56+
57+
TEST(PrintLogTest, LogTestWithoutInit) {
58+
// Without ArrowLog::StartArrowLog, this should also work.
59+
PrintLog();
60+
}
61+
62+
TEST(PrintLogTest, LogTestWithInit) {
63+
// Test empty app name.
64+
ArrowLog::StartArrowLog("", ArrowLogLevel::ARROW_DEBUG);
65+
PrintLog();
66+
ArrowLog::ShutDownArrowLog();
67+
}
68+
69+
// This test will output large amount of logs to stderr, should be disabled in travis.
70+
TEST(LogPerfTest, PerfTest) {
71+
ArrowLog::StartArrowLog("/fake/path/to/appdire/LogPerfTest", ArrowLogLevel::ARROW_ERROR,
72+
"/tmp/");
73+
int rounds = 10000;
74+
75+
int64_t start_time = current_time_ms();
76+
for (int i = 0; i < rounds; ++i) {
77+
ARROW_LOG(DEBUG) << "This is the "
78+
<< "ARROW_DEBUG message";
79+
}
80+
int64_t elapsed = current_time_ms() - start_time;
81+
std::cout << "Testing DEBUG log for " << rounds << " rounds takes " << elapsed << " ms."
82+
<< std::endl;
83+
84+
start_time = current_time_ms();
85+
for (int i = 0; i < rounds; ++i) {
86+
ARROW_LOG(ERROR) << "This is the "
87+
<< "RARROW_ERROR message";
88+
}
89+
elapsed = current_time_ms() - start_time;
90+
std::cout << "Testing ARROW_ERROR log for " << rounds << " rounds takes " << elapsed
91+
<< " ms." << std::endl;
92+
93+
start_time = current_time_ms();
94+
for (int i = 0; i < rounds; ++i) {
95+
ARROW_CHECK(i >= 0) << "This is a ARROW_CHECK "
96+
<< "message but it won't show up";
97+
}
98+
elapsed = current_time_ms() - start_time;
99+
std::cout << "Testing ARROW_CHECK(true) for " << rounds << " rounds takes " << elapsed
100+
<< " ms." << std::endl;
101+
ArrowLog::ShutDownArrowLog();
102+
}
103+
104+
} // namespace arrow
105+
106+
int main(int argc, char** argv) {
107+
::testing::InitGoogleTest(&argc, argv);
108+
return RUN_ALL_TESTS();
109+
}

0 commit comments

Comments
 (0)