From cc72cfd8ba8561da684f550af912256100328fbc Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Thu, 4 Jun 2020 10:33:58 -0700 Subject: [PATCH 1/8] Enabled the VisRTX OptiX ray tracing back end Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/development/Dockerfile | 3 ++- Scripts/docker/ubuntu/runtime/Dockerfile | 2 +- cmake/sites/Docker-Ubuntu-18_04.cmake | 5 +++-- projects/visrtx.cmake | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Scripts/docker/ubuntu/development/Dockerfile b/Scripts/docker/ubuntu/development/Dockerfile index 307e76e2..4e4296f6 100644 --- a/Scripts/docker/ubuntu/development/Dockerfile +++ b/Scripts/docker/ubuntu/development/Dockerfile @@ -50,7 +50,7 @@ # "Release", or "Debug" (Default: "Release") # -ARG BASE_IMAGE=nvidia/opengl:1.0-glvnd-devel-ubuntu18.04 +ARG BASE_IMAGE=nvidia/cudagl:10.1-devel-ubuntu18.04 FROM ${BASE_IMAGE} ARG RENDERING=egl @@ -81,6 +81,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ libtool \ openssl \ + libglu1-mesa-dev \ pkg-config && \ if [ "${PYTHON_VERSION}" = "2" ]; then apt-get install -y --no-install-recommends python2.7-dev python-pip python-setuptools; fi && \ if [ "${PYTHON_VERSION}" = "3" ]; then apt-get install -y --no-install-recommends python3-dev python3-pip python3-setuptools; fi && \ diff --git a/Scripts/docker/ubuntu/runtime/Dockerfile b/Scripts/docker/ubuntu/runtime/Dockerfile index 613ebd03..d16fbfc0 100644 --- a/Scripts/docker/ubuntu/runtime/Dockerfile +++ b/Scripts/docker/ubuntu/runtime/Dockerfile @@ -22,7 +22,7 @@ # . # -ARG BASE_IMAGE=nvidia/opengl:1.0-glvnd-devel-ubuntu18.04 +ARG BASE_IMAGE=nvidia/cudagl:10.1-devel-ubuntu18.04 ARG PV_DEV_BASE_IMAGE=pv-dev-image # Start with the pv development image so we can copy from it diff --git a/cmake/sites/Docker-Ubuntu-18_04.cmake b/cmake/sites/Docker-Ubuntu-18_04.cmake index 44f675d2..f18a742a 100644 --- a/cmake/sites/Docker-Ubuntu-18_04.cmake +++ b/cmake/sites/Docker-Ubuntu-18_04.cmake @@ -25,8 +25,9 @@ if("$ENV{RENDERING}" STREQUAL "egl") set(PARAVIEW_EXTRA_CMAKE_ARGUMENTS "-DOpenGL_GL_PREFERENCE:STRING=GLVND" CACHE STRING "") # If we have support for EGL, index and rtx would be nice eventually too - set(ENABLE_nvidiaindex OFF CACHE BOOL "") - set(ENABLE_visrtx OFF CACHE BOOL "") + set(ENABLE_nvidiaindex ON CACHE BOOL "") + set(ENABLE_nvidiamdl ON CACHE BOOL "") + set(ENABLE_visrtx ON CACHE BOOL "") else() message('Builing with OSMESA rendering') set(ENABLE_osmesa ON CACHE BOOL "") diff --git a/projects/visrtx.cmake b/projects/visrtx.cmake index 2ae4fdfa..af1e876c 100644 --- a/projects/visrtx.cmake +++ b/projects/visrtx.cmake @@ -4,4 +4,5 @@ superbuild_add_project(visrtx -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} -DCMAKE_MACOSX_RPATH:BOOL=FALSE -DCMAKE_INSTALL_NAME_DIR:PATH=/lib - -DCMAKE_INSTALL_LIBDIR:STRING=lib) + -DCMAKE_INSTALL_LIBDIR:STRING=lib + -DVISRTX_BUILD_SAMPLE:BOOL=FALSE) -- GitLab From 1f6c0ed5f4e92833eeb59d4d166d58bdcaf44815 Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Thu, 4 Jun 2020 10:53:55 -0700 Subject: [PATCH 2/8] Specifying the base container as a variable seems to interfere with pulling it Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/development/Dockerfile | 3 +-- Scripts/docker/ubuntu/runtime/Dockerfile | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Scripts/docker/ubuntu/development/Dockerfile b/Scripts/docker/ubuntu/development/Dockerfile index 4e4296f6..f9a6417e 100644 --- a/Scripts/docker/ubuntu/development/Dockerfile +++ b/Scripts/docker/ubuntu/development/Dockerfile @@ -50,8 +50,7 @@ # "Release", or "Debug" (Default: "Release") # -ARG BASE_IMAGE=nvidia/cudagl:10.1-devel-ubuntu18.04 -FROM ${BASE_IMAGE} +FROM nvidia/cudagl:10.1-devel-ubuntu18.04 ARG RENDERING=egl ARG PARAVIEW_TAG=v5.8.0 diff --git a/Scripts/docker/ubuntu/runtime/Dockerfile b/Scripts/docker/ubuntu/runtime/Dockerfile index d16fbfc0..e11952f8 100644 --- a/Scripts/docker/ubuntu/runtime/Dockerfile +++ b/Scripts/docker/ubuntu/runtime/Dockerfile @@ -22,14 +22,8 @@ # . # -ARG BASE_IMAGE=nvidia/cudagl:10.1-devel-ubuntu18.04 -ARG PV_DEV_BASE_IMAGE=pv-dev-image - -# Start with the pv development image so we can copy from it -FROM ${PV_DEV_BASE_IMAGE} AS devimage - -# Now pick the base of that image as our new base -FROM ${BASE_IMAGE} AS baseimage +FROM paraview:5.8-devel-py3-egl AS devimage +FROM nvidia/cudagl:10.1-runtime-ubuntu18.04 AS baseimage # Now install python the same same way we did in the base ARG PYTHON_VERSION=2 @@ -44,6 +38,8 @@ ENV SYSTEM_PYTHON_3_PIP pip3 ENV SYSTEM_PYTHON_PIP "SYSTEM_PYTHON_${PYTHON_VERSION}_PIP" RUN apt-get update && \ + apt install -y --no-install-recommends \ + libgomp1 && \ if [ "${PYTHON_VERSION}" = "2" ]; then apt-get install -y --no-install-recommends python2.7-dev python-pip python-setuptools; fi && \ if [ "${PYTHON_VERSION}" = "3" ]; then apt-get install -y --no-install-recommends python3-dev python3-pip python3-setuptools; fi && \ if [ "${PYTHON_VERSION}" = "3" ]; then update-alternatives --install /usr/bin/python python /usr/bin/python3 1; fi && \ @@ -54,6 +50,13 @@ RUN groupadd pv-user && \ useradd -g pv-user -d /home/pv-user pv-user && \ mkdir /home/pv-user && chown -R pv-user:pv-user /home/pv-user +COPY --from=devimage /usr/lib/i386-linux-gnu/libOpenGL.so /usr/lib/i386-linux-gnu/ +COPY --from=devimage /usr/lib/x86_64-linux-gnu/libOpenGL.so /usr/lib/x86_64-linux-gnu/ +RUN ln -s /usr/lib/i386-linux-gnu/libOpenGL.so /usr/lib/i386-linux-gnu/libOpenGL.so.0 \ + && ln -s /usr/lib/i386-linux-gnu/libOpenGL.so /usr/lib/i386-linux-gnu/libOpenGL.so.0.0 \ + && ln -s /usr/lib/x86_64-linux-gnu/libOpenGL.so /usr/lib/x86_64-linux-gnu/libOpenGL.so.0 \ + && ln -s /usr/lib/x86_64-linux-gnu/libOpenGL.so /usr/lib/x86_64-linux-gnu/libOpenGL.so.0.0 + # Now copy the installed paraview from the development image, using # the non-root user as the owner. COPY --from=devimage --chown=pv-user:pv-user /opt/paraview /opt/paraview -- GitLab From a1d578dc23ce5de728acfae1fbd0a394ab88e0ef Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Thu, 11 Jun 2020 20:45:10 -0700 Subject: [PATCH 3/8] Changed the commit of the superbuild and temporarily changed the remote address Signed-off-by: Nick Leaf --- .gitmodules | 2 +- superbuild | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3e33d45d..519c113a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "superbuild"] path = superbuild - url = https://gitlab.kitware.com/paraview/common-superbuild.git + url = https://gitlab.kitware.com/nleaf/common-superbuild.git diff --git a/superbuild b/superbuild index b3d946f6..3ad1bb6e 160000 --- a/superbuild +++ b/superbuild @@ -1 +1 @@ -Subproject commit b3d946f6caa9ecb85a4a2287b5189dbc300b19b2 +Subproject commit 3ad1bb6e204e0440da0a061373859ee79227fc70 -- GitLab From 753cf9f77e113167ab18f6dd6e92e8ef0528d7d0 Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Thu, 11 Jun 2020 20:54:17 -0700 Subject: [PATCH 4/8] Added a custom OpenMPI build with UCX support, and switched the superbuild to use it by enabling system MPI Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/development/Dockerfile | 108 +++++++++++++++++++ Scripts/docker/ubuntu/runtime/Dockerfile | 22 ++++ cmake/sites/Docker-Ubuntu-18_04.cmake | 8 ++ 3 files changed, 138 insertions(+) diff --git a/Scripts/docker/ubuntu/development/Dockerfile b/Scripts/docker/ubuntu/development/Dockerfile index f9a6417e..bcd5a7cd 100644 --- a/Scripts/docker/ubuntu/development/Dockerfile +++ b/Scripts/docker/ubuntu/development/Dockerfile @@ -81,6 +81,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libtool \ openssl \ libglu1-mesa-dev \ + libnuma-dev \ + librdmacm-dev \ + libpmi2-0-dev \ + flex \ pkg-config && \ if [ "${PYTHON_VERSION}" = "2" ]; then apt-get install -y --no-install-recommends python2.7-dev python-pip python-setuptools; fi && \ if [ "${PYTHON_VERSION}" = "3" ]; then apt-get install -y --no-install-recommends python3-dev python3-pip python3-setuptools; fi && \ @@ -88,6 +92,110 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* && \ ${!SYSTEM_PYTHON_PIP} install mako + +################################################################################ +### +# XPMEM/git:master +### + +RUN cd /tmp \ + && git clone --depth 1 -b master https://gitlab.com/hjelmn/xpmem \ + && cd xpmem \ + && autoreconf --install \ + && ./configure --disable-kernel-module --prefix=/usr/local/xpmem \ + && make \ + && make install + +RUN echo /usr/local/xpmem/lib >> /etc/ld.so.conf.d/nvidia.conf \ + && ldconfig + +### +# KNEM/knem-1.1.3 +### + +RUN cd /tmp \ + && git clone --depth 1 -b knem-1.1.3 https://gforge.inria.fr/git/knem/knem.git \ + && cd knem \ + && mkdir -p /usr/local/knem/include \ + && cp common/knem_io.h /usr/local/knem/include + +RUN echo /usr/local/knem/lib >> /etc/ld.so.conf.d/nvidia.conf \ + && ldconfig + +### +# OFED/apt-get +### + +### +# UCX/v1.8.0 +### + +RUN cd /tmp \ + && mkdir -p ./usr/local/ucx \ + && cd ./usr/local/ucx \ + && git clone --depth 1 -b v1.8.0 https://github.com/openucx/ucx.git \ + && cd ucx \ + && ./autogen.sh \ + && CFLAGS="-Wl,-rpath=None/lib" ./configure --prefix=/usr/local/ucx \ + --enable-compiler-opt=3 \ + --with-avx \ + --disable-doxygen-doc \ + --disable-logging \ + --disable-debug \ + --disable-assertions \ + --disable-params-check \ + --without-cuda \ + --with-knem=/usr/local/knem \ + --with-xpmem=/usr/local/xpmem \ + --with-verbs \ + --with-rdmacm \ + --enable-shared \ + --enable-static=no \ + --enable-cma \ + && make -j $(nproc) all \ + && make install + +RUN echo /usr/local/ucx/lib >> /etc/ld.so.conf.d/nvidia.conf \ + && ldconfig + +### +# OpenMPI/v4.0.3 with UCX and slurm pmi2 support +### + +ENV MPI_DIRECTORY="/usr/local/openmpi" +RUN cd /tmp \ + && cp /usr/include/slurm-wlm/* /usr/include \ + && git clone --depth 1 -b v4.0.3 https://github.com/open-mpi/ompi.git \ + && cd ompi \ + && ./autogen.pl \ + && ./configure --prefix=$MPI_DIRECTORY \ + --with-slurm \ + --with-pmix \ + --with-libevent \ + --with-pmi \ + --disable-oshmem \ + --enable-orterun-prefix-by-default \ + --disable-debug \ + --disable-mem-debug \ + --disable-mem-profile \ + --disable-memchecker \ + --without-cuda \ + --with-ucx=/usr/local/ucx \ + --with-ucx-libdir=/usr/local/ucx/lib \ + --with-hwloc=internal \ + --enable-mpi1-compatibility \ + --enable-shared \ + --disable-static \ + && make -j $(nproc) \ + && make install + +RUN echo /usr/local/openmpi/lib >> /etc/ld.so.conf.d/nvidia.conf \ + && ldconfig +################################################################################ + + + + # Create a non-root user RUN groupadd pv-user && \ useradd -g pv-user -d /home/pv-user pv-user && \ diff --git a/Scripts/docker/ubuntu/runtime/Dockerfile b/Scripts/docker/ubuntu/runtime/Dockerfile index e11952f8..e2109844 100644 --- a/Scripts/docker/ubuntu/runtime/Dockerfile +++ b/Scripts/docker/ubuntu/runtime/Dockerfile @@ -39,6 +39,10 @@ ENV SYSTEM_PYTHON_PIP "SYSTEM_PYTHON_${PYTHON_VERSION}_PIP" RUN apt-get update && \ apt install -y --no-install-recommends \ + numactl \ + librdmacm1 \ + libpmi2-0 \ + ssh \ libgomp1 && \ if [ "${PYTHON_VERSION}" = "2" ]; then apt-get install -y --no-install-recommends python2.7-dev python-pip python-setuptools; fi && \ if [ "${PYTHON_VERSION}" = "3" ]; then apt-get install -y --no-install-recommends python3-dev python3-pip python3-setuptools; fi && \ @@ -57,6 +61,24 @@ RUN ln -s /usr/lib/i386-linux-gnu/libOpenGL.so /usr/lib/i386-linux-gnu/libOpenGL && ln -s /usr/lib/x86_64-linux-gnu/libOpenGL.so /usr/lib/x86_64-linux-gnu/libOpenGL.so.0 \ && ln -s /usr/lib/x86_64-linux-gnu/libOpenGL.so /usr/lib/x86_64-linux-gnu/libOpenGL.so.0.0 +# Copy the installed MPI build from the devimage +ENV UCX_WARN_UNUSED_ENV_VARS=no \ + OMPI_MCA_pml=ucx \ + OMPI_MCA_btl=^openib \ + NVINDEX_PVPLUGIN_HOME=/usr/local/paraview/config \ + PATH=${PATH:+${PATH}:}/usr/local/openmpi/bin:/usr/local/paraview/bin \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}/usr/local/openmpi/lib:/usr/local/paraview/lib:/usr/local/xpmem/lib:/usr/local/ucx/lib \ + PKG_CONFIG_PATH=/usr/local/ucx/lib/pkgconfig:/usr/local/openmpi/lib/pkgconfig:/usr/local/xpmem/lib/pkgconfig +COPY --from=devimage /usr/local/xpmem/lib /usr/local/xpmem/lib +COPY --from=devimage /usr/local/ucx/bin /usr/local/ucx/bin +COPY --from=devimage /usr/local/ucx/lib /usr/local/ucx/lib +COPY --from=devimage /usr/local/openmpi/bin /usr/local/openmpi/bin +COPY --from=devimage /usr/local/openmpi/share /usr/local/openmpi/share +COPY --from=devimage /usr/local/openmpi/lib /usr/local/openmpi/lib +COPY --from=devimage /usr/local/cuda/lib64/libnvrtc.so /usr/local/cuda/lib64/libnvrtc.so +COPY --from=devimage /etc/ld.so.conf.d/nvidia.conf /etc/ld.so.conf.d/nvidia.conf +RUN ldconfig + # Now copy the installed paraview from the development image, using # the non-root user as the owner. COPY --from=devimage --chown=pv-user:pv-user /opt/paraview /opt/paraview diff --git a/cmake/sites/Docker-Ubuntu-18_04.cmake b/cmake/sites/Docker-Ubuntu-18_04.cmake index f18a742a..0449fe31 100644 --- a/cmake/sites/Docker-Ubuntu-18_04.cmake +++ b/cmake/sites/Docker-Ubuntu-18_04.cmake @@ -87,6 +87,7 @@ set(ENABLE_visitbridge ON CACHE BOOL "") set(ENABLE_ffmpeg ON CACHE BOOL "") set(ENABLE_vistrails ON CACHE BOOL "") set(ENABLE_mpi ON CACHE BOOL "") +set(USE_SYSTEM_mpi ON CACHE BOOL "") set(ENABLE_silo ON CACHE BOOL "") set(ENABLE_xdmf3 ON CACHE BOOL "") set(ENABLE_h5py ON CACHE BOOL "") @@ -102,6 +103,13 @@ set(ENABLE_las ON CACHE BOOL "") set(ENABLE_acusolve ON CACHE BOOL "") set(ENABLE_fontconfig ON CACHE BOOL "") +# MPI-related: tell cmake where to find OpenMPI installation +if(USE_SYSTEM_mpi) + set(MPI_C_COMPILER $ENV{MPI_DIRECTORY}/bin/mpicc CACHE STRING "") + set(MPI_CXX_COMPILER $ENV{MPI_DIRECTORY}/bin/mpicxx CACHE STRING "") + set(MPI_Fortran_COMPILER $ENV{MPI_DIRECTORY}/bin/mpifort CACHE STRING "") +endif() + # FIXME: We should be able to have these, but they didn't work at some point set(ENABLE_vrpn OFF CACHE BOOL "") -- GitLab From e8df2ad4196b097ab8b2b3eb32cb4cf06ebc290f Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Fri, 12 Jun 2020 10:13:21 -0700 Subject: [PATCH 5/8] Removed trailing whitespace Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/development/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Scripts/docker/ubuntu/development/Dockerfile b/Scripts/docker/ubuntu/development/Dockerfile index bcd5a7cd..c539032f 100644 --- a/Scripts/docker/ubuntu/development/Dockerfile +++ b/Scripts/docker/ubuntu/development/Dockerfile @@ -94,7 +94,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ################################################################################ -### +### # XPMEM/git:master ### @@ -109,7 +109,7 @@ RUN cd /tmp \ RUN echo /usr/local/xpmem/lib >> /etc/ld.so.conf.d/nvidia.conf \ && ldconfig -### +### # KNEM/knem-1.1.3 ### @@ -122,11 +122,11 @@ RUN cd /tmp \ RUN echo /usr/local/knem/lib >> /etc/ld.so.conf.d/nvidia.conf \ && ldconfig -### +### # OFED/apt-get ### -### +### # UCX/v1.8.0 ### @@ -158,7 +158,7 @@ RUN cd /tmp \ RUN echo /usr/local/ucx/lib >> /etc/ld.so.conf.d/nvidia.conf \ && ldconfig -### +### # OpenMPI/v4.0.3 with UCX and slurm pmi2 support ### -- GitLab From 0405725af889cb129e37d710175e20fe038d43aa Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Mon, 15 Jun 2020 11:38:25 -0700 Subject: [PATCH 6/8] Added ParaView's lib dir to the library search path Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/runtime/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Scripts/docker/ubuntu/runtime/Dockerfile b/Scripts/docker/ubuntu/runtime/Dockerfile index e2109844..6b28802e 100644 --- a/Scripts/docker/ubuntu/runtime/Dockerfile +++ b/Scripts/docker/ubuntu/runtime/Dockerfile @@ -83,5 +83,9 @@ RUN ldconfig # the non-root user as the owner. COPY --from=devimage --chown=pv-user:pv-user /opt/paraview /opt/paraview +# Add ParaView's library directory to the library path +ENV LD_LIBRARY_PATH=/opt/paraview/lib:${LD_LIBRARY_PATH} + + USER pv-user WORKDIR /opt/paraview -- GitLab From c4233fa30c9b7b4e122429a67768072c59160b6c Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Mon, 15 Jun 2020 11:42:10 -0700 Subject: [PATCH 7/8] Changed checkout command to be more robust to git submodule path differences Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/development/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Scripts/docker/ubuntu/development/Dockerfile b/Scripts/docker/ubuntu/development/Dockerfile index c539032f..c34f05aa 100644 --- a/Scripts/docker/ubuntu/development/Dockerfile +++ b/Scripts/docker/ubuntu/development/Dockerfile @@ -213,8 +213,7 @@ env CMAKE_EXECUTABLE /home/pv-user/cmake/3.16.4/bin/cmake # Clone the superbuild RUN mkdir -p /home/pv-user/pvsb/build && cd /home/pv-user/pvsb && \ - git clone --recursive ${SUPERBUILD_REPO} src && \ - cd src && git checkout ${SUPERBUILD_TAG} && git submodule update + git clone --recursive --single-branch --branch ${SUPERBUILD_TAG} ${SUPERBUILD_REPO} src WORKDIR /home/pv-user/pvsb/build -- GitLab From 9da55563e80f0ba12fd423e1cb5116e2a4f73481 Mon Sep 17 00:00:00 2001 From: Nick Leaf Date: Wed, 17 Jun 2020 14:44:32 -0700 Subject: [PATCH 8/8] Switched 'ssh' package to 'openssh-client'; this prevents private encryption keys from being stored in the container Signed-off-by: Nick Leaf --- Scripts/docker/ubuntu/runtime/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/docker/ubuntu/runtime/Dockerfile b/Scripts/docker/ubuntu/runtime/Dockerfile index 6b28802e..425bca12 100644 --- a/Scripts/docker/ubuntu/runtime/Dockerfile +++ b/Scripts/docker/ubuntu/runtime/Dockerfile @@ -42,7 +42,7 @@ RUN apt-get update && \ numactl \ librdmacm1 \ libpmi2-0 \ - ssh \ + openssh-client \ libgomp1 && \ if [ "${PYTHON_VERSION}" = "2" ]; then apt-get install -y --no-install-recommends python2.7-dev python-pip python-setuptools; fi && \ if [ "${PYTHON_VERSION}" = "3" ]; then apt-get install -y --no-install-recommends python3-dev python3-pip python3-setuptools; fi && \ -- GitLab