diff --git a/presto-native-execution/.dockerignore b/presto-native-execution/.dockerignore new file mode 100644 index 0000000000000..fea49c5ae99a0 --- /dev/null +++ b/presto-native-execution/.dockerignore @@ -0,0 +1,4 @@ +*.log +build +_build +target \ No newline at end of file diff --git a/presto-native-execution/docker-compose.yml b/presto-native-execution/docker-compose.yml index 534e2a8d79a95..2a384de04eaf1 100644 --- a/presto-native-execution/docker-compose.yml +++ b/presto-native-execution/docker-compose.yml @@ -19,6 +19,38 @@ services: context: . dockerfile: scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile + ubuntu-native-dev: + # Usage: + # docker compose build ubuntu-native-dev + # podman compose build ubuntu-native-dev + image: presto/prestissimo-dependency:ubuntu-22.04 + build: + args: + # A few files in Velox require significant memory to compile and link. + # Build requires 18GB of memory for 2 threads. + - NUM_THREADS=2 # default value for NUM_THREADS. + - DEPENDENCY_IMAGE=presto/prestissimo-dependency:ubuntu-22.04 + - BASE_IMAGE=ubuntu:22.04 + - OSNAME=ubuntu + - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF + -DPRESTO_ENABLE_PARQUET=ON + -DPRESTO_ENABLE_S3=ON + context: . + dockerfile: scripts/dockerfiles/prestissimo-runtime.dockerfile + target: prestissimo-dev + stdin_open: true + tty: true + volumes: + - ..:/presto + working_dir: /presto/presto-native-execution + environment: + - NUM_THREADS=2 # default value for NUM_THREADS + - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF + -DPRESTO_ENABLE_PARQUET=ON + -DPRESTO_ENABLE_S3=ON + command: > + bash -c "git config --global --add safe.directory /presto && git config --global --add safe.directory /presto/presto-native-execution/velox && ccache -sz -v && exec bash -i" + ubuntu-native-runtime: # Usage: # docker compose build ubuntu-native-runtime @@ -47,6 +79,35 @@ services: context: . dockerfile: scripts/dockerfiles/centos-dependency.dockerfile + centos-native-dev: + # Usage: + # docker compose build centos-native-dev + # podman compose build centos-native-dev + image: presto/prestissimo-dependency:centos9 + build: + args: + # A few files in Velox require significant memory to compile and link. + # Build requires 18GB of memory for 2 threads. + - NUM_THREADS=2 # default value for NUM_THREADS + - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF + -DPRESTO_ENABLE_PARQUET=ON + -DPRESTO_ENABLE_S3=ON + context: . + dockerfile: scripts/dockerfiles/prestissimo-runtime.dockerfile + target: prestissimo-dev + stdin_open: true + tty: true + volumes: + - ..:/presto + working_dir: /presto/presto-native-execution + environment: + - NUM_THREADS=2 # default value for NUM_THREADS + - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF + -DPRESTO_ENABLE_PARQUET=ON + -DPRESTO_ENABLE_S3=ON + command: > + bash -c "echo 'unset LANG' >> /root/.bashrc && git config --global --add safe.directory /presto && git config --global --add safe.directory /presto/presto-native-execution/velox && ccache -sz -v && exec bash -i" + centos-native-runtime: # Usage: # docker compose build centos-native-runtime diff --git a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile index 7ae303f21f27d..b26ad45111ed1 100644 --- a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile @@ -16,19 +16,38 @@ FROM ${DEPENDENCY_IMAGE} as prestissimo-image ARG OSNAME=centos ARG BUILD_TYPE=Release +ARG BUILD_DIR='release' ARG EXTRA_CMAKE_FLAGS='' ARG NUM_THREADS=8 ENV PROMPT_ALWAYS_RESPOND=n +ENV BUILD_ROOT=/presto/presto-native-execution ENV BUILD_BASE_DIR=_build -ENV BUILD_DIR="" +ENV CCACHE_NOHASHDIR=true +ENV CCACHE_BASEDIR=${BUILD_ROOT} -RUN mkdir -p /prestissimo /runtime-libraries -COPY . /prestissimo/ -RUN EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} \ - NUM_THREADS=${NUM_THREADS} make --directory="/prestissimo/" cmake-and-build BUILD_TYPE=${BUILD_TYPE} BUILD_DIR=${BUILD_DIR} BUILD_BASE_DIR=${BUILD_BASE_DIR} -RUN !(LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 ldd /prestissimo/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server | grep "not found") && \ - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 ldd /prestissimo/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server | awk 'NF == 4 { system("cp " $3 " /runtime-libraries") }' +RUN mkdir -p ${BUILD_ROOT} /runtime-libraries +COPY . ${BUILD_ROOT} +RUN EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS}" \ + NUM_THREADS=${NUM_THREADS} make --directory="${BUILD_ROOT}" cmake-and-build BUILD_TYPE=${BUILD_TYPE} BUILD_DIR=${BUILD_DIR} BUILD_BASE_DIR=${BUILD_BASE_DIR} && \ + ccache -sz -v +RUN !(LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 ldd ${BUILD_ROOT}/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server | grep "not found") && \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 ldd ${BUILD_ROOT}/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server | awk 'NF == 4 { system("cp " $3 " /runtime-libraries") }' + +#///////////////////////////////////////////// +# prestissimo-dev +#////////////////////////////////////////////// + +FROM ${DEPENDENCY_IMAGE} as prestissimo-dev + +ENV PROMPT_ALWAYS_RESPOND=n +ENV BUILD_ROOT=/presto/presto-native-execution +ENV BUILD_BASE_DIR=_build +ENV CCACHE_NOHASHDIR="true" +ENV CCACHE_BASEDIR=/presto/presto-native-execution + +COPY --from=prestissimo-image /root/.cache/ccache /root/.cache/ccache +RUN ccache -sz -v #///////////////////////////////////////////// # prestissimo-runtime @@ -37,9 +56,9 @@ RUN !(LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 ldd /pr FROM ${BASE_IMAGE} ENV BUILD_BASE_DIR=_build -ENV BUILD_DIR="" +ARG BUILD_DIR='release' -COPY --chmod=0775 --from=prestissimo-image /prestissimo/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server /usr/bin/ +COPY --chmod=0775 --from=prestissimo-image /presto/presto-native-execution/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server /usr/bin/ COPY --chmod=0775 --from=prestissimo-image /runtime-libraries/* /usr/lib64/prestissimo-libs/ COPY --chmod=0755 ./etc /opt/presto-server/etc COPY --chmod=0775 ./entrypoint.sh /opt/entrypoint.sh diff --git a/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile b/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile index e31cc72739ae4..9898ce8c38aef 100644 --- a/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile @@ -22,6 +22,7 @@ ENV SUDO=" " # are required to avoid tzdata installation # to prompt for region selection. ENV TZ=${tz} +ENV LDFLAGS="-no-pie" RUN mkdir -p /scripts /velox/scripts COPY scripts /scripts