Skip to content

Commit 31a86ee

Browse files
authored
Add db-benchmark (apache#365)
* Add db-benchmark * python lint
1 parent 0911bf3 commit 31a86ee

5 files changed

Lines changed: 684 additions & 0 deletions

File tree

‎benchmarks/db-benchmark/README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<!---
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
20+
# Run db-benchmark
21+
22+
This directory contains scripts for running DataFusion with the https://github.com/h2oai/db-benchmark
23+
24+
## Directions
25+
26+
Run the following from root `arrow-datafusion` directory
27+
28+
```bash
29+
$ docker buildx build -t db-benchmark -f benchmarks/db-benchmark/db-benchmark.dockerfile .
30+
$ docker run --privileged db-benchmark
31+
```
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
FROM ubuntu
19+
ARG DEBIAN_FRONTEND=noninteractive
20+
ARG TARGETPLATFORM
21+
22+
RUN apt-get update && \
23+
apt-get install -y git build-essential
24+
25+
# Install R, curl, and python deps
26+
RUN apt-get -y install --no-install-recommends --no-install-suggests \
27+
ca-certificates software-properties-common gnupg2 gnupg1 \
28+
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
29+
&& add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' \
30+
&& apt-get -y install r-base \
31+
&& apt-get -y install curl \
32+
&& apt-get -y install python3.8 \
33+
&& apt-get -y install python3-pip
34+
35+
# Install R libraries
36+
RUN R -e "install.packages('data.table',dependencies=TRUE, repos='http://cran.rstudio.com/')" \
37+
&& R -e "install.packages('dplyr',dependencies=TRUE, repos='http://cran.rstudio.com/')"
38+
39+
# Install Rust
40+
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
41+
ENV PATH="/root/.cargo/bin:${PATH}"
42+
43+
# Clone db-benchmark and download data
44+
RUN git clone https://github.com/h2oai/db-benchmark \
45+
&& cd db-benchmark \
46+
&& Rscript _data/groupby-datagen.R 1e7 1e2 0 0 \
47+
&& Rscript _data/join-datagen.R 1e7 0 0 0 \
48+
&& mkdir data \
49+
&& mv G1_1e7_1e2_0_0.csv data \
50+
&& mv J1_1e7_1e1_0_0.csv data \
51+
&& mv J1_1e7_1e4_0_0.csv data \
52+
&& mv J1_1e7_1e7_0_0.csv data \
53+
&& mv J1_1e7_NA_0_0.csv data \
54+
&& cd ..
55+
56+
# Clone datafusion-python and build python library
57+
# Not sure if the wheel will be the same on all computers
58+
RUN git clone https://github.com/datafusion-contrib/datafusion-python \
59+
&& cd datafusion-python && git reset --hard 368b50ed9662d5e93c70b539f94cceace685265e \
60+
&& python3 -m pip install pip \
61+
&& python3 -m pip install pandas \
62+
&& python3 -m pip install -r requirements.txt \
63+
&& cd ..
64+
65+
# Copy local arrow-datafusion
66+
COPY . arrow-datafusion
67+
68+
# 1. datafusion-python that builds from datafusion version referenced datafusion-python
69+
RUN cd datafusion-python \
70+
&& maturin build --release \
71+
&& case "${TARGETPLATFORM}" in \
72+
*/amd64) CPUARCH=x86_64 ;; \
73+
*/arm64) CPUARCH=aarch64 ;; \
74+
*) exit 1 ;; \
75+
esac \
76+
# Version will need to be updated in conjunction with datafusion-python version
77+
&& python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \
78+
&& cd ..
79+
80+
# 2. datafusion-python that builds from local datafusion. use this when making local changes to datafusion.
81+
# Currently, as of March 5th 2022, this done not build (i think) because datafusion is being split into multiple crates
82+
# and datafusion-python has not yet been updated to reflect this.
83+
# RUN cd datafusion-python \
84+
# && sed -i '/datafusion =/c\datafusion = { path = "../arrow-datafusion/datafusion", features = ["pyarrow"] }' Cargo.toml \
85+
# && sed -i '/fuzz-utils/d' ../arrow-datafusion/datafusion/Cargo.toml \
86+
# && maturin build --release \
87+
# && case "${TARGETPLATFORM}" in \
88+
# */amd64) CPUARCH=x86_64 ;; \
89+
# */amd64) CPUARCH=aarch64 ;; \
90+
# *) exit 1 ;; \
91+
# esac \
92+
# && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \
93+
# && cd ..
94+
95+
# Make datafusion directory in db-benchmark
96+
RUN mkdir db-benchmark/datafusion \
97+
&& cp ../arrow-datafusion/benchmarks/db-benchmark/groupby-datafusion.py db-benchmark/datafusion \
98+
&& cp ../arrow-datafusion/benchmarks/db-benchmark/join-datafusion.py db-benchmark/datafusion \
99+
&& cp ../arrow-datafusion/benchmarks/db-benchmark/run-bench.sh db-benchmark/ \
100+
&& chmod +x db-benchmark/run-bench.sh
101+
102+
WORKDIR /db-benchmark
103+
104+
RUN ls && ls -al data/
105+
106+
ENTRYPOINT ./run-bench.sh

0 commit comments

Comments
 (0)