Skip to content

Commit 8776295

Browse files
assignUserkou
assignUser
and
kou
authored
ARROW-18042: [Java] Distribute Apple M1 compatible JNI libraries via mavencentral (apache#14472)
Lead-authored-by: Rok Mihevc <rok@mihevc.org> Co-authored-by: Sutou Kouhei <kou@clear-code.com> Co-authored-by: Jacob Wujciak-Jens <jacob@wujciak.de> Co-authored-by: Sutou Kouhei <kou@cozmixng.org> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 98943d9 commit 8776295

17 files changed

Lines changed: 207 additions & 73 deletions

File tree

‎.github/workflows/java_jni.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ concurrency:
4141

4242
permissions:
4343
contents: read
44-
44+
4545
env:
4646
DOCKER_VOLUME_PREFIX: ".docker/"
4747

‎ci/scripts/java_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ fi
9393

9494
if [ "${ARROW_PLASMA}" = "ON" ]; then
9595
pushd ${source_dir}/plasma
96-
${mvn} clean install
96+
${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} clean install
9797
popd
9898
fi
9999

‎ci/scripts/java_jni_macos_build.sh‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,17 @@ set -ex
2121

2222
arrow_dir=${1}
2323
build_dir=${2}
24+
normalized_arch=$(arch)
25+
case ${normalized_arch} in
26+
arm64)
27+
normalized_arch=aarch_64
28+
;;
29+
i386)
30+
normalized_arch=x86_64
31+
;;
32+
esac
2433
# The directory where the final binaries will be stored when scripts finish
25-
dist_dir=${3}
34+
dist_dir=${3}/${normalized_arch}
2635

2736
echo "=== Clear output directories and leftovers ==="
2837
# Clear output directories and leftovers

‎ci/scripts/java_jni_manylinux_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ set -ex
2222
arrow_dir=${1}
2323
build_dir=${2}
2424
# The directory where the final binaries will be stored when scripts finish
25-
dist_dir=${3}
25+
dist_dir=${3}/$(arch)
2626

2727
echo "=== Clear output directories and leftovers ==="
2828
# Clear output directories and leftovers

‎ci/scripts/java_jni_windows_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ set -ex
2222
arrow_dir=${1}
2323
build_dir=${2}
2424
# The directory where the final binaries will be stored when scripts finish
25-
dist_dir=${3}
25+
dist_dir=${3}/x86_64
2626

2727
echo "=== Clear output directories and leftovers ==="
2828
# Clear output directories and leftovers

‎ci/scripts/java_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ if [ "${#projects[@]}" -gt 0 ]; then
6060
if [ "${ARROW_PLASMA}" = "ON" ]; then
6161
pushd ${source_dir}/plasma
6262
java -cp target/test-classes:target/classes \
63-
-Djava.library.path=${java_jni_dist_dir} \
63+
-Djava.library.path=${java_jni_dist_dir}/$(arch) \
6464
org.apache.arrow.plasma.PlasmaClientTest
6565
popd
6666
fi

‎cpp/Brewfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ brew "grpc"
3131
brew "llvm@14"
3232
brew "lz4"
3333
brew "ninja"
34-
brew "numpy"
3534
brew "openssl@1.1"
3635
brew "protobuf"
3736
brew "python"

‎dev/tasks/java-jars/github.yml‎

Lines changed: 65 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ jobs:
2828
{{ macros.github_checkout_arrow()|indent }}
2929
{{ macros.github_install_archery()|indent }}
3030
- name: Build C++ libraries
31+
env:
32+
{{ macros.github_set_sccache_envvars()|indent(8) }}
3133
run: |
3234
archery docker run \
3335
-e ARROW_JAVA_BUILD=OFF \
@@ -48,47 +50,65 @@ jobs:
4850
{% endif %}
4951

5052
build-cpp-macos:
51-
name: Build C++ libraries macOS
52-
runs-on: macos-latest
53+
{% set arch = '${{ matrix.platform.arch }}' %}
54+
name: Build C++ libraries macOS {{ arch }}
55+
runs-on: {{ '${{ matrix.platform.runs_on }}' }}
56+
strategy:
57+
fail-fast: false
58+
matrix:
59+
platform:
60+
- { runs_on: ["macos-latest"], arch: "x86_64"}
61+
- { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "aarch_64" }
5362
env:
5463
MACOSX_DEPLOYMENT_TARGET: "10.13"
5564
steps:
5665
{{ macros.github_checkout_arrow()|indent }}
57-
{{ macros.github_install_archery()|indent }}
66+
- name: Set up Python
67+
if: |
68+
!contains(matrix.platform.runs_on, 'self-hosted')
69+
uses: actions/setup-python@v4
70+
with:
71+
cache: 'pip'
72+
python-version: 3.8
73+
- name: Install Archery
74+
shell: bash
75+
run: pip install -e arrow/dev/archery[all]
5876
- name: Install dependencies
5977
run: |
60-
brew install --overwrite git
6178
brew bundle --file=arrow/cpp/Brewfile
6279
# We want to link aws-sdk-cpp statically but Homebrew's
6380
# aws-sdk-cpp provides only shared library. If we have
6481
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
6582
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
6683
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
6784
brew uninstall aws-sdk-cpp
68-
- name: Setup ccache
69-
run: |
70-
arrow/ci/scripts/ccache_setup.sh
85+
brew bundle --file=arrow/java/Brewfile
7186
- name: Build C++ libraries
87+
env:
88+
{{ macros.github_set_sccache_envvars()|indent(8) }}
7289
run: |
7390
set -e
91+
# make brew Java available to CMake
92+
if [ "{{ arch }}" = "aarch_64" ]; then
93+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
94+
fi
7495
arrow/ci/scripts/java_jni_macos_build.sh \
7596
$GITHUB_WORKSPACE/arrow \
7697
$GITHUB_WORKSPACE/arrow/cpp-build \
7798
$GITHUB_WORKSPACE/arrow/java-dist
7899
- name: Compress into single artifact to keep directory structure
79-
run: tar -cvzf arrow-shared-libs-macos.tar.gz arrow/java-dist/
100+
run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/
80101
- name: Upload artifacts
81102
uses: actions/upload-artifact@v2
82103
with:
83-
name: macos-shared-lib
84-
path: arrow-shared-libs-macos.tar.gz
104+
name: macos-shared-lib-{{ arch }}
105+
path: arrow-shared-libs-macos-{{ arch }}.tar.gz
85106

86107
build-cpp-windows:
87108
name: Build C++ libraries Windows
88109
runs-on: windows-2019
89110
steps:
90111
{{ macros.github_checkout_arrow()|indent }}
91-
{{ macros.github_install_archery()|indent }}
92112
- name: Set up Java
93113
uses: actions/setup-java@v3
94114
with:
@@ -97,8 +117,13 @@ jobs:
97117
- name: Download Timezone Database
98118
shell: bash
99119
run: arrow/ci/scripts/download_tz_database.sh
120+
- name: Install sccache
121+
shell: bash
122+
run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
100123
- name: Build C++ libraries
101124
shell: cmd
125+
env:
126+
{{ macros.github_set_sccache_envvars()|indent(8) }}
102127
run: |
103128
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
104129
REM For ORC
@@ -122,39 +147,42 @@ jobs:
122147
- build-cpp-windows
123148
steps:
124149
{{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
125-
- name: Download Linux C++ Libraries
126-
uses: actions/download-artifact@v2
150+
- name: Download Libraries
151+
uses: actions/download-artifact@v3
127152
with:
128-
name: ubuntu-shared-lib
129-
- name: Download macOS C++ libraries
130-
uses: actions/download-artifact@v2
131-
with:
132-
name: macos-shared-lib
133-
- name: Download Windows C++ libraries
134-
uses: actions/download-artifact@v2
135-
with:
136-
name: windows-shared-lib
137-
- name: Descompress artifacts
153+
path: artifacts
154+
- name: Decompress artifacts
138155
run: |
156+
mv artifacts/*/*.tar.gz .
139157
tar -xvzf arrow-shared-libs-linux.tar.gz
140-
tar -xvzf arrow-shared-libs-macos.tar.gz
158+
tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
159+
tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
141160
tar -xvzf arrow-shared-libs-windows.tar.gz
142161
- name: Test that shared libraries exist
143162
run: |
144163
set -x
145-
test -f arrow/java-dist/libarrow_cdata_jni.dylib
146-
test -f arrow/java-dist/libarrow_cdata_jni.so
147-
test -f arrow/java-dist/arrow_cdata_jni.dll
148-
test -f arrow/java-dist/libarrow_dataset_jni.dylib
149-
test -f arrow/java-dist/libarrow_dataset_jni.so
150-
test -f arrow/java-dist/arrow_dataset_jni.dll
151-
test -f arrow/java-dist/libarrow_orc_jni.dylib
152-
test -f arrow/java-dist/libarrow_orc_jni.so
153-
test -f arrow/java-dist/arrow_orc_jni.dll
154-
test -f arrow/java-dist/libgandiva_jni.dylib
155-
test -f arrow/java-dist/libgandiva_jni.so
156-
test -f arrow/java-dist/libplasma_java.dylib
157-
test -f arrow/java-dist/libplasma_java.so
164+
165+
test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so
166+
test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so
167+
test -f arrow/java-dist/x86_64/libarrow_orc_jni.so
168+
test -f arrow/java-dist/x86_64/libgandiva_jni.so
169+
test -f arrow/java-dist/x86_64/libplasma_java.so
170+
171+
test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib
172+
test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
173+
test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
174+
test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
175+
test -f arrow/java-dist/x86_64/libplasma_java.dylib
176+
177+
test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.dylib
178+
test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib
179+
test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib
180+
test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib
181+
test -f arrow/java-dist/aarch_64/libplasma_java.dylib
182+
183+
test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll
184+
test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll
185+
test -f arrow/java-dist/x86_64/arrow_orc_jni.dll
158186
- name: Build bundled jar
159187
run: |
160188
set -e

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1210,7 +1210,7 @@ services:
12101210
command:
12111211
[ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
12121212
/arrow/ci/scripts/python_build.sh /arrow /build &&
1213-
/arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java &&
1213+
/arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) &&
12141214
/arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
12151215
/arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ]
12161216

‎docs/source/developers/java/building.rst‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ Maven
112112
$ export JAVA_HOME=<absolute path to your java home>
113113
$ java --version
114114
$ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
115-
$ ls -latr ../java-dist/lib
115+
$ ls -latr ../java-dist/lib/<your system's architecture>
116116
|__ libarrow_cdata_jni.dylib
117117
|__ libarrow_cdata_jni.so
118118
@@ -122,7 +122,7 @@ Maven
122122
123123
$ cd arrow/java
124124
$ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
125-
$ dir "../java-dist/bin"
125+
$ dir "../java-dist/bin/x86_64"
126126
|__ arrow_cdata_jni.dll
127127
128128
- To build all JNI libraries (MacOS / Linux) except the JNI C Data Interface library:
@@ -133,7 +133,7 @@ Maven
133133
$ export JAVA_HOME=<absolute path to your java home>
134134
$ java --version
135135
$ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N
136-
$ ls -latr java-dist/lib/*_{jni,java}.*
136+
$ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
137137
|__ libarrow_dataset_jni.dylib
138138
|__ libarrow_orc_jni.dylib
139139
|__ libgandiva_jni.dylib
@@ -145,7 +145,7 @@ Maven
145145
146146
$ cd arrow/java
147147
$ mvn generate-resources -Pgenerate-libs-jni-windows -N
148-
$ dir "../java-dist/bin"
148+
$ dir "../java-dist/bin/x86_64"
149149
|__ arrow_dataset_jni.dll
150150
151151
CMake
@@ -164,7 +164,7 @@ CMake
164164
-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \
165165
-DBUILD_TESTING=OFF \
166166
-DCMAKE_BUILD_TYPE=Release \
167-
-DCMAKE_INSTALL_LIBDIR=lib \
167+
-DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
168168
-DCMAKE_INSTALL_PREFIX=java-dist
169169
$ cmake --build java-cdata --target install --config Release
170170
$ ls -latr java-dist/lib
@@ -184,7 +184,7 @@ CMake
184184
-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^
185185
-DBUILD_TESTING=OFF ^
186186
-DCMAKE_BUILD_TYPE=Release ^
187-
-DCMAKE_INSTALL_LIBDIR=lib ^
187+
-DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
188188
-DCMAKE_INSTALL_PREFIX=java-dist
189189
$ cmake --build java-cdata --target install --config Release
190190
$ dir "java-dist/bin"
@@ -219,7 +219,7 @@ CMake
219219
-DARROW_S3=ON \
220220
-DARROW_USE_CCACHE=ON \
221221
-DCMAKE_BUILD_TYPE=Release \
222-
-DCMAKE_INSTALL_LIBDIR=lib \
222+
-DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
223223
-DCMAKE_INSTALL_PREFIX=java-dist \
224224
-DCMAKE_UNITY_BUILD=ON
225225
$ cmake --build cpp-jni --target install --config Release
@@ -230,11 +230,11 @@ CMake
230230
-DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \
231231
-DBUILD_TESTING=OFF \
232232
-DCMAKE_BUILD_TYPE=Release \
233-
-DCMAKE_INSTALL_LIBDIR=lib \
233+
-DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
234234
-DCMAKE_INSTALL_PREFIX=java-dist \
235235
-DCMAKE_PREFIX_PATH=$PWD/java-dist
236236
$ cmake --build java-jni --target install --config Release
237-
$ ls -latr java-dist/lib/*_{jni,java}.*
237+
$ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
238238
|__ libarrow_dataset_jni.dylib
239239
|__ libarrow_orc_jni.dylib
240240
|__ libgandiva_jni.dylib
@@ -265,7 +265,7 @@ CMake
265265
-DARROW_WITH_ZLIB=ON ^
266266
-DARROW_WITH_ZSTD=ON ^
267267
-DCMAKE_BUILD_TYPE=Release ^
268-
-DCMAKE_INSTALL_LIBDIR=lib ^
268+
-DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
269269
-DCMAKE_INSTALL_PREFIX=java-dist ^
270270
-DCMAKE_UNITY_BUILD=ON ^
271271
-GNinja
@@ -282,7 +282,7 @@ CMake
282282
-DARROW_JAVA_JNI_ENABLE_PLASMA=OFF ^
283283
-DBUILD_TESTING=OFF ^
284284
-DCMAKE_BUILD_TYPE=Release ^
285-
-DCMAKE_INSTALL_LIBDIR=lib ^
285+
-DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
286286
-DCMAKE_INSTALL_PREFIX=java-dist ^
287287
-DCMAKE_PREFIX_PATH=$PWD/java-dist
288288
$ cmake --build java-jni --target install --config Release
@@ -296,7 +296,7 @@ Archery
296296
297297
$ cd arrow
298298
$ archery docker run java-jni-manylinux-2014
299-
$ ls -latr java-dist/
299+
$ ls -latr java-dist/<your system's architecture>/
300300
|__ libarrow_cdata_jni.so
301301
|__ libarrow_dataset_jni.so
302302
|__ libarrow_orc_jni.so

0 commit comments

Comments
 (0)