Skip to content

Commit dbf531b

Browse files
wesmwesm
committed
ARROW-3050: [C++] Adopt HiveServer2 client codebase from
cloudera/hs2client. Add Thrift to thirdparty toolchain This patch incorporates patches developed at cloudera/hs2client (Apache 2.0) by the following authors: * 12 Wes McKinney <wesm@apache.org>, <wes@cloudera.com> * 2 Thomas Tauber-Marshall <tmarshall@cloudera.com> * 2 陈晓发 <chenxiaofa@chelun.com> * 2 Matthew Jacobs <jacobs.mh@gmail.com>, <mj@cloudera.com> * 1 Miki Tebeka <miki.tebeka@gmail.com> * 1 Tim Armstrong <tarmstrong@cloudera.com> * 1 henryr <henry.robinson@gmail.com> Closes apache#2444 Change-Id: I88aed528a9f4d2069a4908f6a09230ade2fbe50a
1 parent 94e8196 commit dbf531b

42 files changed

Lines changed: 6917 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎LICENSE.txt‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -630,3 +630,45 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
630630
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
631631

632632
--------------------------------------------------------------------------------
633+
634+
This project includes code from the hs2client
635+
636+
https://github.com/cloudera/hs2client
637+
638+
Copyright 2016 Cloudera Inc.
639+
640+
Licensed under the Apache License, Version 2.0 (the "License");
641+
you may not use this file except in compliance with the License.
642+
You may obtain a copy of the License at
643+
644+
http://www.apache.org/licenses/LICENSE-2.0
645+
646+
Unless required by applicable law or agreed to in writing, software
647+
distributed under the License is distributed on an "AS IS" BASIS,
648+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
649+
See the License for the specific language governing permissions and
650+
limitations under the License.
651+
652+
--------------------------------------------------------------------------------
653+
654+
The script dev/docker_common/wait-for-it.sh has the following license
655+
656+
Copyright (c) 2016 Giles Hall
657+
658+
Permission is hereby granted, free of charge, to any person obtaining a copy of
659+
this software and associated documentation files (the "Software"), to deal in
660+
the Software without restriction, including without limitation the rights to
661+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
662+
of the Software, and to permit persons to whom the Software is furnished to do
663+
so, subject to the following conditions:
664+
665+
The above copyright notice and this permission notice shall be included in all
666+
copies or substantial portions of the Software.
667+
668+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
669+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
670+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
671+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
672+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
673+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
674+
SOFTWARE.

‎cpp/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,10 @@ Pass multiple labels by dividing with semicolons")
203203
"Build Arrow libraries with install_name set to @rpath"
204204
ON)
205205

206+
option(ARROW_HIVESERVER2
207+
"Build the HiveServer2 client and Arrow adapter"
208+
OFF)
209+
206210
option(ARROW_PLASMA
207211
"Build the plasma object store along with Arrow"
208212
OFF)
@@ -721,3 +725,7 @@ add_subdirectory(src/arrow)
721725
if(ARROW_PYTHON)
722726
add_subdirectory(src/arrow/python)
723727
endif()
728+
729+
if(ARROW_HIVESERVER2)
730+
add_subdirectory(src/arrow/dbi/hiveserver2)
731+
endif()

‎cpp/cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
3131
set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3232
set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3333
set(ZSTD_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
34+
set(THRIFT_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
3435

3536
if (NOT DEFINED ENV{BOOST_ROOT})
3637
# Since we have to set this in the environment, we check whether
@@ -79,6 +80,10 @@ if (DEFINED ENV{PROTOBUF_HOME})
7980
set(PROTOBUF_HOME "$ENV{PROTOBUF_HOME}")
8081
endif()
8182

83+
if (DEFINED ENV{THRIFT_HOME})
84+
set(THRIFT_HOME "$ENV{THRIFT_HOME}")
85+
endif()
86+
8287
# ----------------------------------------------------------------------
8388
# Versions and URLs for toolchain builds, which also can be used to configure
8489
# offline builds
@@ -192,6 +197,12 @@ else()
192197
set(ORC_SOURCE_URL "https://github.com/apache/orc/archive/rel/release-${ORC_VERSION}.tar.gz")
193198
endif()
194199

200+
if (DEFINED ENV{ARROW_THRIFT_URL})
201+
set(THRIFT_SOURCE_URL "$ENV{ARROW_THRIFT_URL}")
202+
else()
203+
set(THRIFT_SOURCE_URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz")
204+
endif()
205+
195206
# ----------------------------------------------------------------------
196207
# ExternalProject options
197208

@@ -1084,3 +1095,136 @@ if (ARROW_ORC)
10841095
endif()
10851096

10861097
endif()
1098+
1099+
# ----------------------------------------------------------------------
1100+
# Thrift
1101+
1102+
if (ARROW_HIVESERVER2)
1103+
1104+
# find thrift headers and libs
1105+
find_package(Thrift)
1106+
1107+
if (NOT THRIFT_FOUND)
1108+
set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install")
1109+
set(ZLIB_HOME "${ZLIB_PREFIX}")
1110+
set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include")
1111+
if (MSVC)
1112+
if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
1113+
set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
1114+
else()
1115+
set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
1116+
endif()
1117+
else()
1118+
set(ZLIB_STATIC_LIB_NAME libz.a)
1119+
endif()
1120+
set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}")
1121+
set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
1122+
-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
1123+
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
1124+
-DBUILD_SHARED_LIBS=OFF)
1125+
ExternalProject_Add(zlib_ep
1126+
URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
1127+
BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
1128+
${ZLIB_BUILD_BYPRODUCTS}
1129+
${EP_LOG_OPTIONS}
1130+
CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
1131+
1132+
set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep/src/thrift_ep-install")
1133+
set(THRIFT_HOME "${THRIFT_PREFIX}")
1134+
set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include")
1135+
set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift")
1136+
set(THRIFT_CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
1137+
"-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
1138+
"-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
1139+
"-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}"
1140+
"-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib"
1141+
"-DBUILD_SHARED_LIBS=OFF"
1142+
"-DBUILD_TESTING=OFF"
1143+
"-DBUILD_EXAMPLES=OFF"
1144+
"-DBUILD_TUTORIALS=OFF"
1145+
"-DWITH_QT4=OFF"
1146+
"-DWITH_C_GLIB=OFF"
1147+
"-DWITH_JAVA=OFF"
1148+
"-DWITH_PYTHON=OFF"
1149+
"-DWITH_HASKELL=OFF"
1150+
"-DWITH_CPP=ON"
1151+
"-DWITH_STATIC_LIB=ON"
1152+
"-DWITH_LIBEVENT=OFF"
1153+
)
1154+
1155+
# Thrift also uses boost. Forward important boost settings if there were ones passed.
1156+
if (DEFINED BOOST_ROOT)
1157+
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}")
1158+
endif()
1159+
if (DEFINED Boost_NAMESPACE)
1160+
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}")
1161+
endif()
1162+
1163+
set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift")
1164+
if (MSVC)
1165+
if (ARROW_USE_STATIC_CRT)
1166+
set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}mt")
1167+
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON")
1168+
else()
1169+
set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}md")
1170+
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF")
1171+
endif()
1172+
endif()
1173+
if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
1174+
set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}d")
1175+
endif()
1176+
set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
1177+
1178+
if (MSVC)
1179+
set(WINFLEXBISON_VERSION 2.4.9)
1180+
set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install")
1181+
ExternalProject_Add(winflexbison_ep
1182+
URL https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip
1183+
URL_HASH MD5=a2e979ea9928fbf8567e995e9c0df765
1184+
SOURCE_DIR ${WINFLEXBISON_PREFIX}
1185+
CONFIGURE_COMMAND ""
1186+
BUILD_COMMAND ""
1187+
INSTALL_COMMAND ""
1188+
${EP_LOG_OPTIONS})
1189+
set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep)
1190+
1191+
set(THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe"
1192+
"-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe"
1193+
"-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}"
1194+
"-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}"
1195+
"-DWITH_SHARED_LIB=OFF"
1196+
"-DWITH_PLUGIN=OFF"
1197+
${THRIFT_CMAKE_ARGS})
1198+
set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} zlib_ep)
1199+
elseif (APPLE)
1200+
if (DEFINED BISON_EXECUTABLE)
1201+
set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${BISON_EXECUTABLE}"
1202+
${THRIFT_CMAKE_ARGS})
1203+
endif()
1204+
endif()
1205+
1206+
ExternalProject_Add(thrift_ep
1207+
URL ${THRIFT_SOURCE_URL}
1208+
BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}"
1209+
CMAKE_ARGS ${THRIFT_CMAKE_ARGS}
1210+
DEPENDS ${THRIFT_DEPENDENCIES}
1211+
${EP_LOG_OPTIONS})
1212+
1213+
set(THRIFT_VENDORED 1)
1214+
else()
1215+
set(THRIFT_VENDORED 0)
1216+
endif()
1217+
1218+
include_directories(SYSTEM ${THRIFT_INCLUDE_DIR} ${THRIFT_INCLUDE_DIR}/thrift)
1219+
message(STATUS "Thrift include dir: ${THRIFT_INCLUDE_DIR}")
1220+
message(STATUS "Thrift static library: ${THRIFT_STATIC_LIB}")
1221+
message(STATUS "Thrift compiler: ${THRIFT_COMPILER}")
1222+
message(STATUS "Thrift version: ${THRIFT_VERSION}")
1223+
add_library(thriftstatic STATIC IMPORTED)
1224+
set_target_properties(thriftstatic PROPERTIES IMPORTED_LOCATION ${THRIFT_STATIC_LIB})
1225+
1226+
if (THRIFT_VENDORED)
1227+
add_dependencies(thriftstatic thrift_ep)
1228+
endif()
1229+
1230+
endif() # ARROW_HIVESERVER2

‎cpp/src/arrow/dbi/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!---
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
20+
# Arrow Database Interfaces
21+
22+
## HiveServer2
23+
24+
For Apache Hive and Apache Impala. See `hiveserver2/` directory

‎cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Headers: top level
19+
install(FILES
20+
api.h
21+
columnar-row-set.h
22+
operation.h
23+
service.h
24+
session.h
25+
types.h
26+
util.h
27+
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/dbi/hiveserver2")
28+
29+
set(ARROW_HIVESERVER2_SRCS
30+
columnar-row-set.cc
31+
service.cc
32+
session.cc
33+
operation.cc
34+
sample-usage.cc
35+
thrift-internal.cc
36+
types.cc
37+
util.cc
38+
)
39+
40+
add_subdirectory(thrift)
41+
42+
set(HIVESERVER2_THRIFT_SRC
43+
ErrorCodes_constants.cpp
44+
ErrorCodes_types.cpp
45+
ImpalaService.cpp
46+
ImpalaService_constants.cpp
47+
ImpalaService_types.cpp
48+
ImpalaHiveServer2Service.cpp
49+
beeswax_constants.cpp
50+
beeswax_types.cpp
51+
BeeswaxService.cpp
52+
TCLIService.cpp
53+
TCLIService_constants.cpp
54+
TCLIService_types.cpp
55+
ExecStats_constants.cpp
56+
ExecStats_types.cpp
57+
hive_metastore_constants.cpp
58+
hive_metastore_types.cpp
59+
Status_constants.cpp
60+
Status_types.cpp
61+
Types_constants.cpp
62+
Types_types.cpp
63+
)
64+
65+
set_source_files_properties(${HIVESERVER2_THRIFT_SRC} PROPERTIES
66+
COMPILE_FLAGS "-Wno-unused-variable -Wno-shadow-field"
67+
GENERATED TRUE)
68+
69+
# keep everything in one library, the object files reference
70+
# each other
71+
add_library(arrow_hiveserver2_thrift STATIC ${HIVESERVER2_THRIFT_SRC})
72+
73+
# Setting these files as code-generated lets make clean and incremental builds work
74+
# correctly
75+
76+
# TODO(wesm): Something is broken with the dependency chain with
77+
# ImpalaService.cpp and others. Couldn't figure out what is different between
78+
# this setup and Impala.
79+
80+
add_dependencies(arrow_hiveserver2_thrift hs2-thrift-cpp)
81+
82+
set_target_properties(arrow_hiveserver2_thrift
83+
PROPERTIES
84+
LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
85+
86+
ADD_ARROW_LIB(arrow_hiveserver2
87+
SOURCES ${ARROW_HIVESERVER2_SRCS}
88+
DEPENDENCIES arrow_hiveserver2_thrift
89+
SHARED_LINK_FLAGS ""
90+
SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS}
91+
)
92+
93+
set_property(SOURCE ${ARROW_HIVESERVER2_SRCS}
94+
APPEND_STRING PROPERTY COMPILE_FLAGS
95+
" -Wno-shadow-field")
96+
97+
set(ARROW_HIVESERVER2_TEST_LINK_LIBS
98+
${ARROW_TEST_LINK_LIBS}
99+
arrow_hiveserver2_static
100+
arrow_hiveserver2_thrift
101+
thriftstatic)
102+
103+
ADD_ARROW_TEST(hiveserver2-test
104+
STATIC_LINK_LIBS "${ARROW_HIVESERVER2_TEST_LINK_LIBS}"
105+
LABELS "hiveserver2"
106+
)
107+
108+
set_property(TARGET hiveserver2-test
109+
APPEND_STRING PROPERTY COMPILE_FLAGS
110+
" -Wno-shadow-field")

‎cpp/src/arrow/dbi/hiveserver2/api.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "arrow/dbi/hiveserver2/columnar-row-set.h"
21+
#include "arrow/dbi/hiveserver2/operation.h"
22+
#include "arrow/dbi/hiveserver2/service.h"
23+
#include "arrow/dbi/hiveserver2/session.h"
24+
#include "arrow/dbi/hiveserver2/types.h"
25+
#include "arrow/dbi/hiveserver2/util.h"
26+
27+
#include "arrow/status.h"

0 commit comments

Comments
 (0)