1+ # Licensed to the Apache Software Foundation (ASF) under one
2+ # or more contributor license agreements. See the NOTICE file
3+ # distributed with this work for additional information
4+ # regarding copyright ownership. The ASF licenses this file
5+ # to you under the Apache License, Version 2.0 (the
6+ # "License"); you may not use this file except in compliance
7+ # with the License. You may obtain a copy of the License at
8+ #
9+ # http://www.apache.org/licenses/LICENSE-2.0
10+ #
11+ # Unless required by applicable law or agreed to in writing,
12+ # software distributed under the License is distributed on an
13+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+ # KIND, either express or implied. See the License for the
15+ # specific language governing permissions and limitations
16+ # under the License.
17+
18+ FROM ubuntu
19+ ARG DEBIAN_FRONTEND=noninteractive
20+ ARG TARGETPLATFORM
21+
22+ RUN apt-get update && \
23+ apt-get install -y git build-essential
24+
25+ # Install R, curl, and python deps
26+ RUN apt-get -y install --no-install-recommends --no-install-suggests \
27+ ca-certificates software-properties-common gnupg2 gnupg1 \
28+ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
29+ && add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' \
30+ && apt-get -y install r-base \
31+ && apt-get -y install curl \
32+ && apt-get -y install python3.8 \
33+ && apt-get -y install python3-pip
34+
35+ # Install R libraries
36+ RUN R -e "install.packages('data.table',dependencies=TRUE, repos='http://cran.rstudio.com/')" \
37+ && R -e "install.packages('dplyr',dependencies=TRUE, repos='http://cran.rstudio.com/')"
38+
39+ # Install Rust
40+ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
41+ ENV PATH="/root/.cargo/bin:${PATH}"
42+
43+ # Clone db-benchmark and download data
44+ RUN git clone https://github.com/h2oai/db-benchmark \
45+ && cd db-benchmark \
46+ && Rscript _data/groupby-datagen.R 1e7 1e2 0 0 \
47+ && Rscript _data/join-datagen.R 1e7 0 0 0 \
48+ && mkdir data \
49+ && mv G1_1e7_1e2_0_0.csv data \
50+ && mv J1_1e7_1e1_0_0.csv data \
51+ && mv J1_1e7_1e4_0_0.csv data \
52+ && mv J1_1e7_1e7_0_0.csv data \
53+ && mv J1_1e7_NA_0_0.csv data \
54+ && cd ..
55+
56+ # Clone datafusion-python and build python library
57+ # Not sure if the wheel will be the same on all computers
58+ RUN git clone https://github.com/datafusion-contrib/datafusion-python \
59+ && cd datafusion-python && git reset --hard 368b50ed9662d5e93c70b539f94cceace685265e \
60+ && python3 -m pip install pip \
61+ && python3 -m pip install pandas \
62+ && python3 -m pip install -r requirements.txt \
63+ && cd ..
64+
65+ # Copy local arrow-datafusion
66+ COPY . arrow-datafusion
67+
68+ # 1. datafusion-python that builds from datafusion version referenced datafusion-python
69+ RUN cd datafusion-python \
70+ && maturin build --release \
71+ && case "${TARGETPLATFORM}" in \
72+ */amd64) CPUARCH=x86_64 ;; \
73+ */arm64) CPUARCH=aarch64 ;; \
74+ *) exit 1 ;; \
75+ esac \
76+ # Version will need to be updated in conjunction with datafusion-python version
77+ && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \
78+ && cd ..
79+
80+ # 2. datafusion-python that builds from local datafusion. use this when making local changes to datafusion.
81+ # Currently, as of March 5th 2022, this done not build (i think) because datafusion is being split into multiple crates
82+ # and datafusion-python has not yet been updated to reflect this.
83+ # RUN cd datafusion-python \
84+ # && sed -i '/datafusion =/c\datafusion = { path = "../arrow-datafusion/datafusion", features = ["pyarrow"] }' Cargo.toml \
85+ # && sed -i '/fuzz-utils/d' ../arrow-datafusion/datafusion/Cargo.toml \
86+ # && maturin build --release \
87+ # && case "${TARGETPLATFORM}" in \
88+ # */amd64) CPUARCH=x86_64 ;; \
89+ # */amd64) CPUARCH=aarch64 ;; \
90+ # *) exit 1 ;; \
91+ # esac \
92+ # && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \
93+ # && cd ..
94+
95+ # Make datafusion directory in db-benchmark
96+ RUN mkdir db-benchmark/datafusion \
97+ && cp ../arrow-datafusion/benchmarks/db-benchmark/groupby-datafusion.py db-benchmark/datafusion \
98+ && cp ../arrow-datafusion/benchmarks/db-benchmark/join-datafusion.py db-benchmark/datafusion \
99+ && cp ../arrow-datafusion/benchmarks/db-benchmark/run-bench.sh db-benchmark/ \
100+ && chmod +x db-benchmark/run-bench.sh
101+
102+ WORKDIR /db-benchmark
103+
104+ RUN ls && ls -al data/
105+
106+ ENTRYPOINT ./run-bench.sh
0 commit comments