Skip to content

Commit 3567dcf

Browse files
koukou
committed
ARROW-8743: [CI][C++] Add a test job for s390x
Closes apache#7131 from kou/cpp-ci-s390x Authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 3752e34 commit 3567dcf

9 files changed

Lines changed: 168 additions & 17 deletions

File tree

‎.travis.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
dist: bionic
19+
20+
language: minimal
21+
22+
addons:
23+
apt:
24+
packages:
25+
- python3-pip
26+
27+
services:
28+
- docker
29+
30+
jobs:
31+
include:
32+
- name: "C++ on s390x"
33+
os: linux
34+
arch: s390x
35+
env:
36+
ARCH: s390x
37+
ARROW_CI_MODULES: "CPP"
38+
ARROW_COMPUTE: "OFF"
39+
ARROW_CSV: "OFF"
40+
ARROW_DATASET: "OFF"
41+
ARROW_FILESYSTEM: "OFF"
42+
ARROW_FLIGHT: "OFF"
43+
ARROW_GANDIVA: "OFF"
44+
ARROW_HDFS: "OFF"
45+
ARROW_IPC: "OFF"
46+
ARROW_ORC: "OFF"
47+
ARROW_PARQUET: "OFF"
48+
ARROW_PLASMA: "OFF"
49+
PARQUET_BUILD_EXAMPLES: "OFF"
50+
PARQUET_BUILD_EXECUTABLES: "OFF"
51+
DOCKER_IMAGE_ID: ubuntu-cpp
52+
UBUNTU: "20.04"
53+
allow_failures:
54+
- arch: s390x
55+
56+
env:
57+
DOCKER_BUILDKIT: 0
58+
COMPOSE_DOCKER_CLI_BUILD: 1
59+
ARROW_ENABLE_TIMING_TESTS: "OFF"
60+
61+
before_install:
62+
- eval "$(python ci/detect-changes.py)"
63+
- |
64+
arrow_ci_affected=no
65+
for arrow_ci_module in ${ARROW_CI_MODULES}; do
66+
arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED
67+
if [ "$(eval 'echo ${arrow_ci_affected_variable}')" != "1" ]; then
68+
arrow_ci_affected=yes
69+
fi
70+
done
71+
if [ "${arrow_ci_affected}" = "no" ]; then
72+
travis_terminate 0
73+
fi
74+
75+
install:
76+
- pip3 install -e dev/archery[docker]
77+
78+
script:
79+
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
80+
# This isn't allowed on Travis CI:
81+
# /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted
82+
- |
83+
ulimit -c unlimited || :
84+
- |
85+
archery docker run \
86+
-e ARROW_COMPUTE=${ARROW_COMPUTE:-ON} \
87+
-e ARROW_CSV=${ARROW_CSV:-ON} \
88+
-e ARROW_DATASET=${ARROW_DATASET:-ON} \
89+
-e ARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \
90+
-e ARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \
91+
-e ARROW_GANDIVA=${ARROW_GANDIVA:-ON} \
92+
-e ARROW_HDFS=${ARROW_HDFS:-ON} \
93+
-e ARROW_IPC=${ARROW_IPC:-ON} \
94+
-e ARROW_ORC=${ARROW_ORC:-ON} \
95+
-e ARROW_PARQUET=${ARROW_PARQUET:-ON} \
96+
-e ARROW_PLASMA=${ARROW_PLASMA:-ON} \
97+
-e PARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-ON} \
98+
-e PARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-ON} \
99+
--volume ${PWD}/build:/build \
100+
${DOCKER_IMAGE_ID}
101+
102+
after_success:
103+
- |
104+
archery docker push ${DOCKER_IMAGE_ID} || :

‎ci/docker/ubuntu-20.04-cpp.dockerfile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ RUN apt-get update -y -q && \
6666
libssl-dev \
6767
libthrift-dev \
6868
libzstd-dev \
69+
make \
6970
ninja-build \
7071
pkg-config \
7172
protobuf-compiler \
@@ -104,6 +105,7 @@ ENV ARROW_BUILD_TESTS=ON \
104105
ARROW_WITH_ZSTD=ON \
105106
GTest_SOURCE=BUNDLED \
106107
ORC_SOURCE=BUNDLED \
107-
PARQUET_BUILD_EXECUTABLES=ON \
108108
PARQUET_BUILD_EXAMPLES=ON \
109-
PATH=/usr/lib/ccache/:$PATH
109+
PARQUET_BUILD_EXECUTABLES=ON \
110+
PATH=/usr/lib/ccache/:$PATH \
111+
PYTHON=python3

‎cpp/build-support/run-test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ function run_test() {
9797
rm -f $XMLFILE
9898

9999
$TEST_EXECUTABLE "$@" 2>&1 \
100-
| $ROOT/build-support/asan_symbolize.py \
100+
| ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \
101101
| ${CXXFILT:-c++filt} \
102102
| $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \
103103
| $pipe_cmd 2>&1 | tee $LOGFILE

‎cpp/cmake_modules/SetupCxxFlags.cmake‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "armv7")
2929
set(ARROW_CPU_FLAG "armv7")
3030
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc")
3131
set(ARROW_CPU_FLAG "ppc")
32+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
33+
set(ARROW_CPU_FLAG "s390x")
3234
endif()
3335
# Check architecture specific compiler flags
3436
if(ARROW_CPU_FLAG STREQUAL "x86")

‎dev/archery/archery/cli.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
#!/usr/bin/env python3
21
# Licensed to the Apache Software Foundation (ASF) under one
32
# or more contributor license agreements. See the NOTICE file
43
# distributed with this work for additional information
@@ -696,9 +695,11 @@ def docker_compose(obj, src):
696695
@click.option('--dry-run/--execute', default=False,
697696
help="Display the docker-compose commands instead of executing "
698697
"them.")
698+
@click.option('--volume', '-v', multiple=True,
699+
help="Set volume within the container")
699700
@click.pass_obj
700701
def docker_compose_run(obj, image, command, env, force_pull, force_build,
701-
use_cache, use_leaf_cache, dry_run):
702+
use_cache, use_leaf_cache, dry_run, volume):
702703
"""Execute docker-compose builds.
703704
704705
To see the available builds run `archery docker list`.
@@ -723,6 +724,9 @@ def docker_compose_run(obj, image, command, env, force_pull, force_build,
723724
# pass runtime parameters via docker environment variables
724725
archery docker run -e CMAKE_BUILD_TYPE=release ubuntu-cpp
725726
727+
# set a volume
728+
archery docker run -v $PWD/build:/build ubuntu-cpp
729+
726730
# starting an interactive bash session for debugging
727731
archery docker run ubuntu-cpp bash
728732
"""
@@ -749,7 +753,8 @@ def _print_command(self, *args, **kwargs):
749753
force_pull=force_pull,
750754
force_build=force_build,
751755
use_cache=use_cache,
752-
use_leaf_cache=use_leaf_cache
756+
use_leaf_cache=use_leaf_cache,
757+
volumes=volume
753758
)
754759
except UndefinedImage as e:
755760
raise click.ClickException(

‎dev/archery/archery/docker.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
#!/usr/bin/env python3
21
# Licensed to the Apache Software Foundation (ASF) under one
32
# or more contributor license agreements. See the NOTICE file
43
# distributed with this work for additional information
@@ -162,7 +161,8 @@ def build(self, image, use_cache=True, use_leaf_cache=True):
162161
self._execute('build', '--no-cache', image)
163162

164163
def run(self, image, command=None, env=None, force_pull=False,
165-
force_build=False, use_cache=True, use_leaf_cache=True):
164+
force_build=False, use_cache=True, use_leaf_cache=True,
165+
volumes=None):
166166
self._validate_image(image)
167167

168168
if force_pull:
@@ -175,6 +175,9 @@ def run(self, image, command=None, env=None, force_pull=False,
175175
if env is not None:
176176
for k, v in env.items():
177177
args.extend(['-e', '{}={}'.format(k, v)])
178+
if volumes is not None:
179+
for volume in volumes:
180+
args.extend(['--volume', volume])
178181

179182
args.append(image)
180183
if command is not None:

‎dev/archery/archery/tests/test_cli.py‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@
3434
force_pull=True,
3535
force_build=True,
3636
use_cache=True,
37-
use_leaf_cache=True
37+
use_leaf_cache=True,
38+
volumes=()
3839
)
3940
),
4041
(
@@ -46,7 +47,8 @@
4647
force_pull=True,
4748
force_build=True,
4849
use_cache=True,
49-
use_leaf_cache=True
50+
use_leaf_cache=True,
51+
volumes=()
5052
)
5153
),
5254
(
@@ -58,7 +60,8 @@
5860
force_pull=False,
5961
force_build=False,
6062
use_cache=True,
61-
use_leaf_cache=True
63+
use_leaf_cache=True,
64+
volumes=()
6265
)
6366
),
6467
(
@@ -73,11 +76,12 @@
7376
force_pull=False,
7477
force_build=True,
7578
use_cache=False,
76-
use_leaf_cache=False
79+
use_leaf_cache=False,
80+
volumes=()
7781
)
7882
),
7983
(
80-
['-e', 'ARROW_GANDIVA=OFF', '-e', 'ARROW_FLIGHT=ON', 'ubuntu-cpp'],
84+
['-e', 'ARROW_GANDIVA=OFF', '-e', 'ARROW_FLIGHT=ON', 'ubuntu-cpp'],
8185
['ubuntu-cpp'],
8286
dict(
8387
command=None,
@@ -88,7 +92,27 @@
8892
force_pull=True,
8993
force_build=True,
9094
use_cache=True,
91-
use_leaf_cache=True
95+
use_leaf_cache=True,
96+
volumes=()
97+
)
98+
),
99+
(
100+
[
101+
'--volume', './build:/build', '-v', './ccache:/ccache:delegated',
102+
'ubuntu-cpp'
103+
],
104+
['ubuntu-cpp'],
105+
dict(
106+
command=None,
107+
env={},
108+
force_pull=True,
109+
force_build=True,
110+
use_cache=True,
111+
use_leaf_cache=True,
112+
volumes=(
113+
'./build:/build',
114+
'./ccache:/ccache:delegated',
115+
)
92116
)
93117
)
94118
])

‎dev/archery/archery/tests/test_docker.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,17 @@ def test_compose_run(arrow_compose_path):
384384
])
385385
compose.run('conda-python', env=env)
386386

387+
expected_calls = [
388+
(
389+
"run --rm --volume /host/build:/build "
390+
"--volume /host/ccache:/ccache:delegated conda-python"
391+
)
392+
]
393+
compose = DockerCompose(arrow_compose_path)
394+
with assert_compose_calls(compose, expected_calls):
395+
volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated")
396+
compose.run('conda-python', volumes=volumes)
397+
387398

388399
def test_compose_run_force_pull_and_build(arrow_compose_path):
389400
compose = DockerCompose(arrow_compose_path)

‎docker-compose.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
# defaults are set in .env file.
2323
#
2424
# Example:
25-
# $ ARCH=arm64v8 docker-compose build ubuntu-bionic-cpp
26-
# $ ARCH=arm64v8 docker-compose run ubuntu-bionic-cpp
25+
# $ ARCH=arm64v8 docker-compose build ubuntu-cpp
26+
# $ ARCH=arm64v8 docker-compose run ubuntu-cpp
2727
#
2828
#
2929
# Coredumps
@@ -257,7 +257,7 @@ services:
257257
# docker-compose build ubuntu-cpp
258258
# docker-compose run --rm ubuntu-cpp
259259
# Parameters:
260-
# ARCH: amd64, arm64v8, ...
260+
# ARCH: amd64, arm64v8, s390x, ...
261261
# UBUNTU: 16.04, 18.04, 20.04
262262
image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
263263
build:

0 commit comments

Comments
 (0)