Skip to content

Commit 35252cd

Browse files
author
Weiming Zhao
committed
Refactor Dockerfile
1 parent 529c34d commit 35252cd

File tree

4 files changed

+110
-66
lines changed

4 files changed

+110
-66
lines changed

‎.github/actions/build/build_in_docker.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash -xe
22

33
REPO="registry-intl.us-west-1.aliyuncs.com/computation/halo"
4-
VER="0.7.6"
4+
VER="0.7.7"
55
FLAVOR="devel"
66

77
MOUNT_DIR="$PWD"

‎utils/docker/Dockerfile

+106-62
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,103 @@
1+
# syntax=docker/dockerfile:experimental
12
# Build this image: docker build -t halo:[version] .
23

34
ARG BASE_IMAGE
5+
6+
# cmake
7+
FROM ubuntu:18.04 AS cmake
8+
ARG CMAKE_VERSION=3.14.5
9+
RUN apt-get update && apt-get install -y curl gcc g++ make zlib1g zlib1g-dev libcurl4-openssl-dev git && \
10+
gcc --version && \
11+
mkdir /tmp/cmake && \
12+
cd /tmp/cmake && \
13+
curl -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -o cmake.tar.gz && \
14+
tar zxf cmake.tar.gz && \
15+
cd cmake-${CMAKE_VERSION} && \
16+
./bootstrap --system-curl --parallel=48 && \
17+
make -j all && \
18+
make install DESTDIR=/tmp/cmake/install && \
19+
make install && \
20+
tar -C /tmp/cmake/install -cf /tmp/cmake.tar usr && \
21+
rm -fr /tmp/cmake
22+
23+
# binutils
24+
FROM cmake AS binutils
25+
ARG BINUTILS_VERSION=2.35
26+
RUN mkdir /tmp/binutils && \
27+
cd /tmp/binutils && \
28+
curl -s http://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.gz | tar xvz && \
29+
cd binutils-${BINUTILS_VERSION} && \
30+
./configure && \
31+
make -j all && \
32+
make install DESTDIR=/tmp/binutils/install && \
33+
tar -C /tmp/binutils/install -cf /tmp/binutils.tar usr && \
34+
rm -rf /tmp/binutils
35+
36+
# valgrind
37+
FROM cmake AS valgrind
38+
ARG VALGRIND_VERSION=3.18.1
39+
RUN mkdir /tmp/valgrind && \
40+
cd /tmp/valgrind && \
41+
curl -o valgrind.tar.bz2 ftp://sourceware.org/pub/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2 && \
42+
tar jxf valgrind.tar.bz2 && \
43+
cd valgrind-${VALGRIND_VERSION} && \
44+
./configure && \
45+
make -j all && \
46+
make install DESTDIR=/tmp/valgrind/install && \
47+
tar -C /tmp/valgrind/install -cf /tmp/valgrid.tar usr && \
48+
rm -rf /tmp/valgrind
49+
50+
# Build Protobuf (static)
51+
FROM cmake AS pb
52+
RUN git -C /tmp clone --depth=1 https://github.com/protocolbuffers/protobuf.git -b v3.9.1 && \
53+
cd /tmp/protobuf/cmake && \
54+
cmake -G "Unix Makefiles" -Dprotobuf_BUILD_TESTS=OFF \
55+
-Dprotobuf_BUILD_SHARED_LIBS=OFF \
56+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON . && \
57+
make -j && make install DESTDIR=/tmp/protobuf/install && \
58+
tar -C /tmp/protobuf/install -cf /tmp/protobuf.tar usr && \
59+
rm -fr /tmp/protobuf
60+
61+
# Build Flatbuffer
62+
FROM cmake as fb
63+
RUN git -C /tmp clone --depth=1 https://github.com/google/flatbuffers.git -b v1.12.0 && \
64+
cd /tmp/flatbuffers && \
65+
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DFLATBUFFERS_BUILD_SHAREDLIB=OFF && \
66+
make -j && make install DESTDIR=/tmp/flatbuffers/install && \
67+
tar -C /tmp/flatbuffers/install -cf /tmp/flatbuffers.tar usr && \
68+
rm -fr /tmp/flatbuffers
69+
70+
# Build glog
71+
FROM cmake AS glog
72+
RUN git -C /tmp clone --depth=1 https://github.com/google/glog.git -b v0.4.0 && \
73+
cd /tmp/glog && \
74+
cmake -H. -Bbuild -G "Unix Makefiles" && \
75+
cd build && \
76+
make -j && make install DESTDIR=/tmp/glog/install && \
77+
tar -C /tmp/glog/install -cf /tmp/glog.tar usr && \
78+
rm -fr /tmp/glog
79+
80+
# Build DNNL
81+
FROM cmake as dnnl
82+
RUN git -C /tmp clone --depth=1 https://github.com/oneapi-src/oneDNN.git --branch v1.7 && \
83+
cd /tmp/oneDNN && \
84+
cmake -G "Unix Makefiles" -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF -DDNNL_ENABLE_PRIMITIVE_CACHE=ON -DCMAKE_INSTALL_PREFIX=/opt/dnnl && \
85+
make -j && make install DESTDIR=/tmp/oneDNN/install && \
86+
tar -C /tmp/oneDNN/install -cf /tmp/dnnl.tar opt && \
87+
rm -fr /tmp/oneDNN
88+
89+
# Build XNNPack
90+
FROM cmake as xnnpack
91+
RUN git -C /tmp clone https://github.com/google/XNNPACK.git && \
92+
cd /tmp/XNNPACK && git checkout -b tmp 90db69f681ea9abd1ced813c17c00007f14ce58b && \
93+
mkdir /tmp/xnn_build && cd /tmp/xnn_build && \
94+
cmake -G "Unix Makefiles" ../XNNPACK -DXNNPACK_LIBRARY_TYPE=static -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
95+
-DXNNPACK_BUILD_TESTS=OFF -DXNNPACK_BUILD_BENCHMARKS=OFF -DCMAKE_BUILD_TYPE=Release \
96+
-DCMAKE_INSTALL_PREFIX=/opt/XNNPACK && \
97+
make -j && make install DESTDIR=/tmp/XNNPACK/install && \
98+
tar -C /tmp/XNNPACK/install -cf /tmp/xnnpack.tar opt && \
99+
rm -fr /tmp/XNNPACK /mp/xnn_build
100+
4101
FROM ${BASE_IMAGE}
5102

6103
# Redeclare the argument
@@ -125,58 +222,19 @@ RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
125222
RUN pip3 install wheel numpy six jupyter enum34 mock h5py pillow
126223

127224
# Update binutils
128-
ARG BINUTILS_VERSION=2.35
129-
RUN mkdir /tmp/binutils && \
130-
cd /tmp/binutils && \
131-
wget http://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.gz && \
132-
tar zxf binutils-${BINUTILS_VERSION}.tar.gz && \
133-
cd binutils-${BINUTILS_VERSION} && \
134-
./configure && \
135-
make -j all && \
136-
make install && \
137-
rm -rf /tmp/binutils
225+
RUN --mount=from=binutils,target=/pkg,source=/tmp tar -C / -xf /pkg/binutils.tar
138226

139227
# Install cmake
140-
ARG CMAKE_VERSION=3.14.5
141-
RUN mkdir /tmp/cmake && \
142-
cd /tmp/cmake && \
143-
wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz && \
144-
tar zxf cmake-${CMAKE_VERSION}.tar.gz && \
145-
cd cmake-${CMAKE_VERSION} && \
146-
./bootstrap --system-curl --parallel=48 && \
147-
make -j all && \
148-
make install && \
149-
rm -rf /tmp/cmake
228+
RUN --mount=from=cmake,target=/pkg,source=/tmp tar -C / -xf /pkg/cmake.tar
150229

151230
# Install valgrind
152-
ARG VALGRIND_VERSION=3.13.0
153-
RUN mkdir /tmp/valgrind && \
154-
cd /tmp/valgrind && \
155-
wget ftp://sourceware.org/pub/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2 && \
156-
tar jxf valgrind-${VALGRIND_VERSION}.tar.bz2 && \
157-
cd valgrind-${VALGRIND_VERSION} && \
158-
./configure && \
159-
make -j all && \
160-
make install && \
161-
rm -rf /tmp/valgrind
231+
RUN --mount=from=valgrind,target=/pkg,source=/tmp tar -C / -xf /pkg/valgrid.tar
162232

163233
# INSTALL Protobuf (static)
164-
RUN cd /tmp && \
165-
git clone --depth=1 https://github.com/protocolbuffers/protobuf.git -b v3.9.1 && \
166-
cd protobuf/cmake && \
167-
cmake -G Ninja . -Dprotobuf_BUILD_TESTS=OFF \
168-
-Dprotobuf_BUILD_SHARED_LIBS=OFF \
169-
-DCMAKE_POSITION_INDEPENDENT_CODE=ON && \
170-
ninja install && \
171-
rm -fr /tmp/protobuf
234+
RUN --mount=from=pb,target=/pkg,source=/tmp tar -C / -xf /pkg/protobuf.tar
172235

173236
# INSTALL glog
174-
RUN cd /tmp && \
175-
git clone --depth=1 https://github.com/google/glog.git -b v0.4.0 && \
176-
cd glog && \
177-
cmake -H. -Bbuild -G "Unix Makefiles" && cmake --build build && \
178-
cmake --build build --target install && ldconfig && \
179-
rm -fr /tmp/glog
237+
RUN --mount=from=glog,target=/pkg,source=/tmp tar -C / -xf /pkg/glog.tar
180238

181239
# Install GCC-10
182240
RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
@@ -185,35 +243,21 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
185243
apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
186244

187245
# Build & Install DNNL (MKLDNN)
188-
RUN cd /tmp && git clone --depth=1 https://github.com/oneapi-src/oneDNN.git --branch v1.7 && \
189-
cd /tmp/oneDNN && \
190-
cmake -G Ninja -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF -DDNNL_ENABLE_PRIMITIVE_CACHE=ON -DCMAKE_INSTALL_PREFIX=/opt/dnnl && \
191-
ninja install
246+
RUN --mount=from=dnnl,target=/pkg,source=/tmp tar -C / -xf /pkg/dnnl.tar
192247

193248
# Install Parallel
194249
RUN apt-get update && \
195250
apt-get install -y parallel --no-install-recommends && \
196251
apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
197252

198253
# Install Eigen
199-
RUN cd /tmp && wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.bz2 && \
200-
tar jxvf eigen-3.4.0.tar.bz2 && mv eigen-3.4.0 /opt
254+
RUN curl -s https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.bz2 | tar -C /opt -xvj
201255

202256
# Install XNNPack
203-
RUN cd /tmp && git clone https://github.com/google/XNNPACK.git && \
204-
cd /tmp/XNNPACK && git checkout -b tmp 90db69f681ea9abd1ced813c17c00007f14ce58b && \
205-
mkdir /tmp/xnn_build_static && cd /tmp/xnn_build_static && \
206-
cmake -G Ninja ../XNNPACK -DXNNPACK_LIBRARY_TYPE=static -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
207-
-DXNNPACK_BUILD_TESTS=OFF -DXNNPACK_BUILD_BENCHMARKS=OFF -DCMAKE_BUILD_TYPE=Release \
208-
-DCMAKE_INSTALL_PREFIX=/opt/XNNPACK && \
209-
ninja install
257+
RUN --mount=from=xnnpack,target=/pkg,source=/tmp tar -C / -xf /pkg/xnnpack.tar
210258

211259
# Install Flatbuffer
212-
RUN cd /tmp && \
213-
git clone --depth=1 https://github.com/google/flatbuffers.git -b v1.12.0 && \
214-
cd flatbuffers && \
215-
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DFLATBUFFERS_BUILD_SHAREDLIB=ON && make -j && make install && \
216-
rm -fr /tmp/flatbuffers
260+
RUN --mount=from=fb,target=/pkg,source=/tmp tar -C / -xf /pkg/flatbuffers.tar
217261

218262
# INSATLL ONEAPI
219263
RUN if [[ ! "${BASE_IMAGE}" =~ "nvidia" ]]; then wget https://registrationcenter-download.intel.com/akdlm/irc_nas/17769/l_BaseKit_p_2021.2.0.2883_offline.sh && \

‎utils/docker/build_image.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash -xe
22

3-
VER="0.7.6"
3+
VER="0.7.7"
44
FLAVOR="devel"
55
NAMESPACE="registry-intl.us-west-1.aliyuncs.com/computation"
66

@@ -10,6 +10,6 @@ base_image_cpu="ubuntu:18.04"
1010
#docker build --build-arg BASE_IMAGE=${base_image_cpu} \
1111
# -t $NAMESPACE/halo:$VER-$FLAVOR-x86_64-ubuntu18.04 .
1212

13-
docker build --build-arg BASE_IMAGE=${base_image_gpu} \
13+
DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=${base_image_gpu} \
1414
-t $NAMESPACE/halo:$VER-$FLAVOR-cuda11.4.2-cudnn8-ubuntu18.04 .
1515

‎utils/docker/start_docker_cpu.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash -xe
22

3-
VER="0.7.6"
3+
VER="0.7.7"
44
FLAVOR="devel"
55
NAMESPACE="registry-intl.us-west-1.aliyuncs.com/computation"
66

0 commit comments

Comments
 (0)