From b12db5d3c7fe15e7d95a3c62642cabbdea27665a Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Thu, 15 Jan 2026 11:58:25 -0500 Subject: [PATCH 1/6] mpi4py CI with ASAN in separate workflow Address sanitizer helps us catch memory bugs even if they don't manifest into faults right away. The instrumention incurs some overhead so this is run on a reduced set of mpi4py runs. Also tests `ompi_info` and `mpicc`. Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py_asan.yaml | 159 ++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 .github/workflows/ompi_mpi4py_asan.yaml diff --git a/.github/workflows/ompi_mpi4py_asan.yaml b/.github/workflows/ompi_mpi4py_asan.yaml new file mode 100644 index 00000000000..95a519604b4 --- /dev/null +++ b/.github/workflows/ompi_mpi4py_asan.yaml @@ -0,0 +1,159 @@ +name: mpi4py (ASAN) + +on: + pull_request: + workflow_dispatch: + inputs: + repository: + description: 'mpi4py repository' + default: 'mpi4py/mpi4py' + required: false + type: string + ref: + description: 'mpi4py branch/tag/SHA' + default: 'master' + required: false + type: string + +permissions: + contents: read + +jobs: + test: + # We need Unbuntu 24.04 (over 22.04) due to a kernel bug, + # see https://github.com/google/sanitizers/issues/856. + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + MPI4PY_TEST_SPAWN: true + # disable ASAN while building + ASAN_OPTIONS: verify_asan_link_order=0,detect_odr_violation=0,abort_on_error=0 + # disable leak detection + LSAN_OPTIONS: detect_leaks=0,exitcode=0 + + steps: + - name: Configure hostname + run: echo 127.0.0.1 `hostname` | sudo tee -a /etc/hosts > /dev/null + if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} + + - name: Install depencencies + run: sudo apt-get install -y -q + libnuma-dev libasan8 + if: ${{ runner.os == 'Linux' }} + + - name: Checkout Open MPI + uses: actions/checkout@v4 + with: + path: mpi-build + submodules: recursive + + - name: Bootstrap Open MPI + run: ./autogen.pl + working-directory: mpi-build + + # Install into a separate directory (/opt/openmpi) so that we can + # bundle up that tree into an artifact to share with other jobs in + # this github action. Specifically don't use /usr/local, because + # there's a bunch of other stuff already installed in /usr/local, + # and we don't need to include that in our artifact. + - name: Configure Open MPI + run: ./configure + --disable-dependency-tracking + --disable-sphinx + --disable-mpi-fortran + --disable-oshmem + --disable-silent-rules + --prefix=/opt/openmpi + CFLAGS="-O2 -fno-omit-frame-pointer -g -fsanitize=address" + LDFLAGS="-Wl,-rpath,/opt/openmpi/lib -fsanitize=address" + working-directory: mpi-build + + - name: Build MPI + run: make -j $(nproc) + working-directory: mpi-build + + - name: Install MPI + run: sudo make install + working-directory: mpi-build + + - name: Add Open MPI to PATH + run: echo /opt/openmpi/bin >> $GITHUB_PATH + + - name: Tweak MPI + run: | + # Tweak MPI + mca_params="$HOME/.openmpi/mca-params.conf" + mkdir -p "$(dirname "$mca_params")" + echo mpi_param_check = true >> "$mca_params" + echo mpi_show_handle_leaks = true >> "$mca_params" + mca_params="$HOME/.prte/mca-params.conf" + mkdir -p "$(dirname "$mca_params")" + echo rmaps_default_mapping_policy = :oversubscribe >> "$mca_params" + + - name: Show MPI + run: ompi_info + + - name: Show MPICC + run: mpicc -show + + - name: Use Python + uses: actions/setup-python@v5 + with: + python-version: 3 + architecture: x64 + + - name: Install Python packages (build) + run: python -m pip install --upgrade + setuptools pip wheel + + - name: Install Python packages (test) + run: python -m pip install --upgrade + numpy cffi pyyaml + + - name: Checkout mpi4py + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository || 'mpi4py/mpi4py' }} + ref: ${{ inputs.ref }} + + - name: Install mpi4py + run: python -m pip install . + env: + CFLAGS: "-O0" + + - name: Setting up ASAN environment + # LD_PRELOAD is needed to make sure ASAN is the first thing loaded + # as it will otherwise complain. + # Leak detection is currently disabled because of the size of the report. + # The patcher is disabled because ASAN fails if code mmaps data at fixed + # memory addresses, see https://github.com/open-mpi/ompi/issues/12819. + # ODR violation detection is disabled until #13469 is fixed + # Disabling stack use after return detection to reduce slowdown, per + # https://github.com/llvm/llvm-project/issues/64190. + run: | + echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV + echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1,detect_stack_use_after_return=0 >> $GITHUB_ENV + echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV + echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV + + - name: Test mpi4py (singleton) + run: python test/main.py -v -x TestExcErrhandlerNull + if: ${{ true }} + timeout-minutes: 10 + + - name: Test mpi4py (np=1) + run: mpiexec -n 1 python test/main.py -v -x TestExcErrhandlerNull + if: ${{ true }} + timeout-minutes: 10 + + - name: Test mpi4py (np=4) + run: mpiexec -n 4 python test/main.py -v -f -x TestExcErrhandlerNull + if: ${{ true }} + timeout-minutes: 10 + + - name: Show MPI (ASAN) + run: ompi_info + + - name: Show MPICC (ASAN) + run: mpicc -show + From d4ff6ff0cc62d53ed3f900b7eab89b407684893a Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Thu, 15 Jan 2026 12:37:58 -0500 Subject: [PATCH 2/6] Reduce optimization levels and enable ASAN when building mpi4py Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py_asan.yaml | 28 ++++++++----------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ompi_mpi4py_asan.yaml b/.github/workflows/ompi_mpi4py_asan.yaml index 95a519604b4..ad859fb1e71 100644 --- a/.github/workflows/ompi_mpi4py_asan.yaml +++ b/.github/workflows/ompi_mpi4py_asan.yaml @@ -64,7 +64,7 @@ jobs: --disable-oshmem --disable-silent-rules --prefix=/opt/openmpi - CFLAGS="-O2 -fno-omit-frame-pointer -g -fsanitize=address" + CFLAGS="-O1 -fno-omit-frame-pointer -g -fsanitize=address" LDFLAGS="-Wl,-rpath,/opt/openmpi/lib -fsanitize=address" working-directory: mpi-build @@ -90,12 +90,6 @@ jobs: mkdir -p "$(dirname "$mca_params")" echo rmaps_default_mapping_policy = :oversubscribe >> "$mca_params" - - name: Show MPI - run: ompi_info - - - name: Show MPICC - run: mpicc -show - - name: Use Python uses: actions/setup-python@v5 with: @@ -116,11 +110,6 @@ jobs: repository: ${{ inputs.repository || 'mpi4py/mpi4py' }} ref: ${{ inputs.ref }} - - name: Install mpi4py - run: python -m pip install . - env: - CFLAGS: "-O0" - - name: Setting up ASAN environment # LD_PRELOAD is needed to make sure ASAN is the first thing loaded # as it will otherwise complain. @@ -136,6 +125,14 @@ jobs: echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV + - name: Show MPI + run: ompi_info + + - name: Install mpi4py + run: python -m pip install . + env: + CFLAGS: "-O0" + - name: Test mpi4py (singleton) run: python test/main.py -v -x TestExcErrhandlerNull if: ${{ true }} @@ -150,10 +147,3 @@ jobs: run: mpiexec -n 4 python test/main.py -v -f -x TestExcErrhandlerNull if: ${{ true }} timeout-minutes: 10 - - - name: Show MPI (ASAN) - run: ompi_info - - - name: Show MPICC (ASAN) - run: mpicc -show - From 6b8dfeb0f2e67aedc9a68b63988ce5709aaa3076 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Thu, 15 Jan 2026 12:51:49 -0500 Subject: [PATCH 3/6] Enable leak check, for sanity checking Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py_asan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ompi_mpi4py_asan.yaml b/.github/workflows/ompi_mpi4py_asan.yaml index ad859fb1e71..e5bfc17e970 100644 --- a/.github/workflows/ompi_mpi4py_asan.yaml +++ b/.github/workflows/ompi_mpi4py_asan.yaml @@ -122,7 +122,7 @@ jobs: run: | echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1,detect_stack_use_after_return=0 >> $GITHUB_ENV - echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV + echo LSAN_OPTIONS=detect_leaks=1,exitcode=0 >> $GITHUB_ENV echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV - name: Show MPI From 3fbedbf31cf8e10e796be83cdea7ade39afa3799 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Sat, 17 Jan 2026 08:45:51 +0900 Subject: [PATCH 4/6] Disable LSAN and enable stack-use-after-return checks Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py_asan.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ompi_mpi4py_asan.yaml b/.github/workflows/ompi_mpi4py_asan.yaml index e5bfc17e970..5204dba825e 100644 --- a/.github/workflows/ompi_mpi4py_asan.yaml +++ b/.github/workflows/ompi_mpi4py_asan.yaml @@ -121,8 +121,8 @@ jobs: # https://github.com/llvm/llvm-project/issues/64190. run: | echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV - echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1,detect_stack_use_after_return=0 >> $GITHUB_ENV - echo LSAN_OPTIONS=detect_leaks=1,exitcode=0 >> $GITHUB_ENV + echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1 >> $GITHUB_ENV + echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV - name: Show MPI From e12e230968e47fbf338f359a663c532fb5f961b0 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Sat, 17 Jan 2026 09:10:47 +0900 Subject: [PATCH 5/6] ASAN: Configure Open MPI with --enable-debug Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py_asan.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ompi_mpi4py_asan.yaml b/.github/workflows/ompi_mpi4py_asan.yaml index 5204dba825e..3f20cd675df 100644 --- a/.github/workflows/ompi_mpi4py_asan.yaml +++ b/.github/workflows/ompi_mpi4py_asan.yaml @@ -58,6 +58,7 @@ jobs: # and we don't need to include that in our artifact. - name: Configure Open MPI run: ./configure + --enable-debug --disable-dependency-tracking --disable-sphinx --disable-mpi-fortran @@ -121,7 +122,7 @@ jobs: # https://github.com/llvm/llvm-project/issues/64190. run: | echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV - echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1 >> $GITHUB_ENV + echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1,detect_stack_use_after_return=1 >> $GITHUB_ENV echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV From f71172c966f371ea6cfaa77e2c351cd7670ae3c3 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Sat, 17 Jan 2026 10:30:41 +0900 Subject: [PATCH 6/6] Have ompi_info print all variables Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py_asan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ompi_mpi4py_asan.yaml b/.github/workflows/ompi_mpi4py_asan.yaml index 3f20cd675df..a14cb084b0b 100644 --- a/.github/workflows/ompi_mpi4py_asan.yaml +++ b/.github/workflows/ompi_mpi4py_asan.yaml @@ -127,7 +127,7 @@ jobs: echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV - name: Show MPI - run: ompi_info + run: ompi_info --all --all - name: Install mpi4py run: python -m pip install .