Skip to content

Commit 526c3c2

Browse files
caisqcaisq
authored
Merge pull request #6169 from caisq/r0.12-py3-docker
Docker build: add support for python3 (#6030)
2 parents f65c8ce + ebb3b6f commit 526c3c2

File tree

6 files changed

+99
-86
lines changed

6 files changed

+99
-86
lines changed

‎tensorflow/tools/ci_build/update_version.sh‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,6 @@ check_existence file "${CMAKE_SETUP_PY}"
9090

9191
sed -i -e "s/^\_VERSION = [\'\"].*-cmake-experimental[\'\"]/\_VERSION = \'${MAJOR}.${MINOR}.${PATCH}-cmake-experimental\'/g" "${CMAKE_SETUP_PY}"
9292

93-
# Update Dockerfiles in tensorflow/tools/docker/
94-
TOOLS_DOCKER_DIR="${TF_SRC_DIR}/tools/docker"
95-
check_existence dir "${TOOLS_DOCKER_DIR}"
96-
97-
# Determine the files that need to be modified
98-
DOCKERFILES=$(grep -lrE "^ENV TENSORFLOW_VERSION .+" ${TOOLS_DOCKER_DIR})
99-
for DOCKERF in ${DOCKERFILES}; do
100-
sed -i -r -e "s/^ENV TENSORFLOW_VERSION .+/ENV TENSORFLOW_VERSION ${MAJOR}.${MINOR}.${PATCH}/g" "${DOCKERF}"
101-
done
102-
10393

10494
# Update os_setup.md
10595
OS_SETUP="${TF_SRC_DIR}/g3doc/get_started/os_setup.md"

‎tensorflow/tools/docker/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ RUN pip --no-cache-dir install \
3333
&& \
3434
python -m ipykernel.kernelspec
3535

36-
ENV TENSORFLOW_VERSION 0.12.0-rc1
37-
3836
# --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- #
3937
# These lines will be edited automatically by parameterized_docker_build.sh. #
4038
# COPY _PIP_FILE_ /
@@ -43,9 +41,11 @@ ENV TENSORFLOW_VERSION 0.12.0-rc1
4341

4442
# Install TensorFlow CPU version from central repo
4543
RUN pip --no-cache-dir install \
46-
http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-${TENSORFLOW_VERSION}-cp27-none-linux_x86_64.whl
44+
http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.0.0-cp27-none-linux_x86_64.whl
4745
# --- ~ DO NOT EDIT OR DELETE BETWEEN THE LINES --- #
4846

47+
# RUN ln -s /usr/bin/python3 /usr/bin/python#
48+
4949
# Set up our notebook config.
5050
COPY jupyter_notebook_config.py /root/.jupyter/
5151

‎tensorflow/tools/docker/Dockerfile.devel‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,12 @@ WORKDIR /tensorflow
8989
# more difficult to experiment with local changes. Instead, just add
9090
# the built directory to the path.
9191

92+
ENV CI_BUILD_PYTHON python
93+
9294
RUN tensorflow/tools/ci_build/builds/configured CPU \
9395
bazel build -c opt tensorflow/tools/pip_package:build_pip_package && \
9496
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
95-
pip install --upgrade /tmp/pip/tensorflow-*.whl && \
97+
pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
9698
rm -rf /tmp/pip && \
9799
rm -rf /root/.cache
98100
# Clean up pip wheel and Bazel cache when done.

‎tensorflow/tools/docker/Dockerfile.devel-gpu‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
1111
libpng12-dev \
1212
libzmq3-dev \
1313
pkg-config \
14-
python \
1514
python-dev \
1615
rsync \
1716
software-properties-common \
@@ -87,14 +86,15 @@ RUN git clone https://github.com/tensorflow/tensorflow.git && \
8786
WORKDIR /tensorflow
8887

8988
# Configure the build for our CUDA configuration.
89+
ENV CI_BUILD_PYTHON python
9090
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
9191
ENV TF_NEED_CUDA 1
9292
ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2
9393

9494
RUN tensorflow/tools/ci_build/builds/configured GPU \
9595
bazel build -c opt --config=cuda tensorflow/tools/pip_package:build_pip_package && \
9696
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
97-
pip install --upgrade /tmp/pip/tensorflow-*.whl && \
97+
pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
9898
rm -rf /tmp/pip && \
9999
rm -rf /root/.cache
100100
# Clean up pip wheel and Bazel cache when done.

‎tensorflow/tools/docker/Dockerfile.gpu‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ RUN pip --no-cache-dir install \
3333
&& \
3434
python -m ipykernel.kernelspec
3535

36-
ENV TENSORFLOW_VERSION 0.12.0-rc1
37-
3836
# --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- #
3937
# These lines will be edited automatically by parameterized_docker_build.sh. #
4038
# COPY _PIP_FILE_ /
@@ -43,9 +41,11 @@ ENV TENSORFLOW_VERSION 0.12.0-rc1
4341

4442
# Install TensorFlow GPU version.
4543
RUN pip --no-cache-dir install \
46-
http://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${TENSORFLOW_VERSION}-cp27-none-linux_x86_64.whl
44+
http://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.0.0-cp27-none-linux_x86_64.whl
4745
# --- ~ DO NOT EDIT OR DELETE BETWEEN THE LINES --- #
4846

47+
# RUN ln -s /usr/bin/python3 /usr/bin/python#
48+
4949
# Set up our notebook config.
5050
COPY jupyter_notebook_config.py /root/.jupyter/
5151

‎tensorflow/tools/docker/parameterized_docker_build.sh‎

Lines changed: 88 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@
3131
#
3232
# TF_DOCKER_BUILD_CENTRAL_PIP
3333
# (Optional)
34-
# If set to any non-0 and non-empty value, will attempt to use the PIP file
35-
# located on the central repo, instead of locally built pip files.
36-
# This option takes effect only for non-devel builds.
34+
# If set to a non-empty string, will use it as the URL from which the
35+
# pip wheel file will be downloaded (instead of building the pip locally).
3736
#
3837
# TF_DOCKER_BUILD_IMAGE_NAME:
3938
# (Optional)
@@ -81,7 +80,6 @@ mark_check_failed() {
8180

8281
TF_DOCKER_BUILD_TYPE=$(to_lower ${TF_DOCKER_BUILD_TYPE})
8382
TF_DOCKER_BUILD_IS_DEVEL=$(to_lower ${TF_DOCKER_BUILD_IS_DEVEL})
84-
TF_DOCKER_BUILD_CENTRAL_PIP=$(to_lower ${TF_DOCKER_BUILD_CENTRAL_PIP})
8583
TF_DOCKER_BUILD_PYTHON_VERSION=$(to_lower ${TF_DOCKER_BUILD_PYTHON_VERSION:-PYTHON2})
8684
TF_DOCKER_BUILD_OPTIONS=$(to_lower ${TF_DOCKER_BUILD_OPTIONS:-OPT})
8785

@@ -144,6 +142,15 @@ else
144142
"${TF_DOCKER_BUILD_TYPE}"
145143
fi
146144

145+
if [[ "${TF_DOCKER_BUILD_PYTHON_VERSION}" == "python2" ]]; then
146+
:
147+
elif [[ "${TF_DOCKER_BUILD_PYTHON_VERSION}" == "python3" ]]; then
148+
FINAL_TAG="${FINAL_TAG}-py3"
149+
else
150+
die "Unrecognized value in TF_DOCKER_BUILD_PYTHON_VERSION: "\
151+
"${TF_DOCKER_BUILD_PYTHON_VERSION}"
152+
fi
153+
147154
# Verify that the original Dockerfile exists
148155
ORIG_DOCKERFILE="${SCRIPT_DIR}/${ORIG_DOCKERFILE}"
149156
if [[ ! -f "${ORIG_DOCKERFILE}" ]]; then
@@ -156,21 +163,6 @@ echo "FINAL_TAG: ${FINAL_TAG}"
156163
echo "Original Dockerfile: ${ORIG_DOCKERFILE}"
157164
echo ""
158165

159-
160-
DO_PIP_BUILD=0
161-
if [[ ${TF_DOCKER_BUILD_IS_DEVEL} == "yes" ]]; then
162-
# Devel builds has pip build instructions in the Dockerfile
163-
:
164-
else
165-
if [[ ! -z ${TF_DOCKER_BUILD_CENTRAL_PIP} ]] &&
166-
[[ ${TF_DOCKER_BUILD_CENTRAL_PIP} != "0" ]]; then
167-
:
168-
else
169-
DO_PIP_BUILD=1
170-
fi
171-
fi
172-
173-
174166
# Create tmp directory for Docker build
175167
TMP_DIR=$(mktemp -d)
176168
echo ""
@@ -179,67 +171,96 @@ echo "Docker build will occur in temporary directory: ${TMP_DIR}"
179171
# Copy all files to tmp directory for Docker build
180172
cp -r ${SCRIPT_DIR}/* "${TMP_DIR}/"
181173

182-
183-
if [[ "${DO_PIP_BUILD}" == "1" ]]; then
174+
if [[ "${TF_DOCKER_BUILD_IS_DEVEL}" == "no" ]]; then
184175
DOCKERFILE="${TMP_DIR}/Dockerfile"
185176

186-
# Perform local build of the required PIP whl file
187-
export TF_BUILD_CONTAINER_TYPE=${TF_DOCKER_BUILD_TYPE}
188-
export TF_BUILD_PYTHON_VERSION=${TF_DOCKER_BUILD_PYTHON_VERSION}
189-
export TF_BUILD_OPTIONS=${TF_DOCKER_BUILD_OPTIONS}
190-
export TF_BUILD_IS_PIP="PIP"
191-
192-
if [[ "${TF_DOCKER_BUILD_TYPE}" == "gpu" ]]; then
193-
export TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS=\
194-
"${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS} -e TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2"
195-
fi
196-
197-
pushd "${SCRIPT_DIR}/../../../"
198-
rm -rf pip_test/whl &&
199-
tensorflow/tools/ci_build/ci_parameterized_build.sh
200-
PIP_BUILD_EXIT_CODE=$?
201-
popd
202-
203-
# Was the pip build successful?
204-
if [[ ${PIP_BUILD_EXIT_CODE} != "0" ]]; then
205-
die "FAIL: Failed to build pip file locally"
206-
fi
207-
208-
PIP_WHL=$(ls pip_test/whl/*.whl | head -1)
209-
if [[ -z "${PIP_WHL}" ]]; then
210-
die "ERROR: Cannot locate the locally-built pip whl file"
211-
fi
212-
echo "Locally-built PIP whl file is at: ${PIP_WHL}"
213-
214-
# Copy the pip file to tmp directory
215-
cp "${PIP_WHL}" "${TMP_DIR}/" || \
216-
die "ERROR: Failed to copy wheel file: ${PIP_WHL}"
217-
218-
# Use string replacement to put the correct file name into the Dockerfile
219-
PIP_WHL=$(basename "${PIP_WHL}")
220-
221-
# Modify the non-devel Dockerfile to point to the correct pip whl file
222-
# location
223-
sed -e "/# --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- #/,"\
177+
if [[ -z "${TF_DOCKER_BUILD_CENTRAL_PIP}" ]]; then
178+
# Perform local build of the required PIP whl file
179+
export TF_BUILD_CONTAINER_TYPE=${TF_DOCKER_BUILD_TYPE}
180+
export TF_BUILD_PYTHON_VERSION=${TF_DOCKER_BUILD_PYTHON_VERSION}
181+
export TF_BUILD_OPTIONS=${TF_DOCKER_BUILD_OPTIONS}
182+
export TF_BUILD_IS_PIP="PIP"
183+
184+
if [[ "${TF_DOCKER_BUILD_TYPE}" == "gpu" ]]; then
185+
export TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS=\
186+
"${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS} -e TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2"
187+
fi
188+
189+
pushd "${SCRIPT_DIR}/../../../"
190+
rm -rf pip_test/whl &&
191+
tensorflow/tools/ci_build/ci_parameterized_build.sh
192+
PIP_BUILD_EXIT_CODE=$?
193+
popd
194+
195+
# Was the pip build successful?
196+
if [[ ${PIP_BUILD_EXIT_CODE} != "0" ]]; then
197+
die "FAIL: Failed to build pip file locally"
198+
fi
199+
200+
PIP_WHL=$(ls pip_test/whl/*.whl | head -1)
201+
if [[ -z "${PIP_WHL}" ]]; then
202+
die "ERROR: Cannot locate the locally-built pip whl file"
203+
fi
204+
echo "Locally-built PIP whl file is at: ${PIP_WHL}"
205+
206+
# Copy the pip file to tmp directory
207+
cp "${PIP_WHL}" "${TMP_DIR}/" || \
208+
die "ERROR: Failed to copy wheel file: ${PIP_WHL}"
209+
210+
# Use string replacement to put the correct file name into the Dockerfile
211+
PIP_WHL=$(basename "${PIP_WHL}")
212+
213+
# Modify the non-devel Dockerfile to point to the correct pip whl file
214+
# location
215+
sed -e "/# --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- #/,"\
224216
"/# --- ~ DO NOT EDIT OR DELETE BETWEEN THE LINES --- #/c"\
225217
"COPY ${PIP_WHL} /\n"\
226218
"RUN pip --no-cache-dir install /${PIP_WHL}" "${ORIG_DOCKERFILE}" \
227219
> "${DOCKERFILE}"
220+
else
221+
echo "Downloading pip wheel from: ${TF_DOCKER_BUILD_CENTRAL_PIP}"
222+
echo
223+
224+
# Modify the non-devel Dockerfile to point to the correct pip whl URL.
225+
sed -e "/# --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- #/,"\
226+
"/# --- ~ DO NOT EDIT OR DELETE BETWEEN THE LINES --- #/c"\
227+
"RUN pip --no-cache-dir install ${TF_DOCKER_BUILD_CENTRAL_PIP}" "${ORIG_DOCKERFILE}" \
228+
> "${DOCKERFILE}"
229+
fi
228230

229231
echo "Modified Dockerfile at: ${DOCKERFILE}"
232+
echo
233+
234+
# Modify python/pip version if necessary.
235+
if [[ "${TF_DOCKER_BUILD_PYTHON_VERSION}" == "python3" ]]; then
236+
sed -i -e 's/python /python3 /g' "${DOCKERFILE}" && \
237+
sed -i -e 's/python-dev/python3-dev/g' "${DOCKERFILE}" && \
238+
sed -i -e 's/pip /pip3 /g' "${DOCKERFILE}" && \
239+
sed -i -e 's^# RUN ln -s /usr/bin/python3 /usr/bin/python#^RUN ln -s /usr/bin/python3 /usr/bin/python^' "${DOCKERFILE}" && \
240+
echo "Modified Dockerfile for python version "\
241+
"${TF_DOCKER_BUILD_PYTHON_VERSION} at: ${DOCKERFILE}" || \
242+
die "FAILED to modify ${DOCKERFILE} for python3"
243+
fi
230244
else
231-
if [[ "${TF_DOCKER_BUILD_IS_DEVEL}" == "yes" ]]; then
232-
DOCKERFILE="${TMP_DIR}/Dockerfile"
245+
DOCKERFILE="${TMP_DIR}/Dockerfile"
233246

234-
# Modify the devel Dockerfile to specify the git branch
235-
sed -r "s/([\s]*git checkout )(.*)/\1${TF_DOCKER_BUILD_DEVEL_BRANCH}/g" \
236-
"${ORIG_DOCKERFILE}" > "${DOCKERFILE}"
237-
else
238-
DOCKERFILE="${TMP_DIR}/"$(basename "${ORIG_DOCKERFILE}")
247+
# Modify the devel Dockerfile to specify the git branch
248+
sed -r "s/([\s]*git checkout )(.*)/\1${TF_DOCKER_BUILD_DEVEL_BRANCH}/g" \
249+
"${ORIG_DOCKERFILE}" > "${DOCKERFILE}"
250+
251+
# Modify python/pip version if necessary.
252+
if [[ "${TF_DOCKER_BUILD_PYTHON_VERSION}" == "python3" ]]; then
253+
sed -i -e 's/python-dev/python-dev python3-dev/g' "${DOCKERFILE}" && \
254+
sed -i -e 's/python /python3 /g' "${DOCKERFILE}" && \
255+
sed -i -e 's^/tmp/pip^/tmp/pip3^g' "${DOCKERFILE}" && \
256+
sed -i -e 's/pip /pip3 /g' "${DOCKERFILE}" && \
257+
sed -i -e 's/ENV CI_BUILD_PYTHON python/ENV CI_BUILD_PYTHON python3/g' "${DOCKERFILE}" && \
258+
sed -i -e 's^# RUN ln -s /usr/bin/python3 /usr/bin/python#^RUN ln -s /usr/bin/python3 /usr/bin/python^' "${DOCKERFILE}" && \
259+
echo "Modified Dockerfile further for python version ${TF_DOCKER_BUILD_PYTHON_VERSION} at: ${DOCKERFILE}" || \
260+
die "FAILED to modify ${DOCKERFILE} for python3"
239261
fi
240262
fi
241263

242-
243264
# Perform docker build
244265
# Intermediate image name with tag
245266
IMG="${USER}/tensorflow:${FINAL_TAG}"

0 commit comments

Comments
 (0)