Fix the Spack CI segfault, and the test failures it was masking #3683

Workflow file for this run

	name: CI (with Spack)

	on:
	# Uncomment the below 'push' to trigger on push
	push:
	branches:
	- release
	pull_request:
	branches:
	- main
	workflow_dispatch:

	env:
	GITHUB_USER: ${{ github.actor }}
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	OMPI_ALLOW_RUN_AS_ROOT: 1
	OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
	PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe

	jobs:
	ci-spack-build:
	name: ci-spack-build (${{ matrix.mpi }}${{ matrix.petsc && format(', petsc {0}/{1}', matrix.scalar, matrix.index) \|\| ', no petsc' }})
	runs-on: ubuntu-26.04
	container: ubuntu:26.04
	permissions:
	contents: read
	packages: write
	strategy:
	fail-fast: false
	# PETSc scalar/index combinations are covered on OpenMPI; MPICH
	# repeats the two most common ones, as it exercises the MPI layer
	# rather than the scalar type. The PETSc-free build tests the
	# optional-PETSc code paths and needs only one MPI.
	matrix:
	include:
	# PETSc solver sets mirror the per-arch '--download-' packages in
	# docker/Dockerfile.test-env, so this workflow and ccpp.yml build
	# PETSc the same way. Two of those packages have no Spack variant
	# in petsc: 'spai' (real64-32) and serial 'superlu' (all int32
	# arches); everything else matches. Note the pattern the
	# Dockerfile encodes: hypre is double-precision only, SuperLU_DIST
	# has no single-precision complex, and the int64 arches drop metis.
	- {mpi: openmpi, petsc: true, scalar: float32, index: int32,
	petsc_dep: "^petsc+mpi+metis+mumps+ptscotch+scalapack+superlu-dist~hypre~suite-sparse~double~complex~int64"}
	- {mpi: openmpi, petsc: true, scalar: float64, index: int32,
	petsc_dep: "^petsc+mpi+hypre+metis+mumps+ptscotch+scalapack+suite-sparse+superlu-dist+double~complex~int64"}
	- {mpi: openmpi, petsc: true, scalar: complex64, index: int32,
	petsc_dep: "^petsc+mpi+metis+mumps+ptscotch+scalapack~hypre~suite-sparse~superlu-dist~double+complex~int64"}
	- {mpi: openmpi, petsc: true, scalar: complex128, index: int32,
	petsc_dep: "^petsc+mpi+hypre+metis+mumps+ptscotch+scalapack+suite-sparse+superlu-dist+double+complex~int64"}
	- {mpi: openmpi, petsc: true, scalar: float64, index: int64,
	petsc_dep: "^petsc+mpi+hypre+mumps+ptscotch+scalapack+suite-sparse+superlu-dist~metis+double~complex+int64"}
	- {mpi: openmpi, petsc: true, scalar: complex128, index: int64,
	petsc_dep: "^petsc+mpi+hypre+mumps+ptscotch+scalapack+suite-sparse+superlu-dist~metis+double+complex+int64"}
	- {mpi: mpich, petsc: true, scalar: float64, index: int32,
	petsc_dep: "^petsc+mpi+hypre+metis+mumps+ptscotch+scalapack+suite-sparse+superlu-dist+double~complex~int64"}
	- {mpi: mpich, petsc: true, scalar: complex128, index: int32,
	petsc_dep: "^petsc+mpi+hypre+metis+mumps+ptscotch+scalapack+suite-sparse+superlu-dist+double+complex~int64"}
	- {mpi: openmpi, petsc: false}

	steps:
	- name: Branch and commit SHA
	run: \|
	echo "Branch and commit SHA: ${{ github.event_name == 'pull_request' && github.head_ref \|\| github.ref_name }}, ${{ github.sha }}"

	- name: Install Spack requirements
	run: \|
	apt-get -y update
	apt-get install -y --no-install-recommends file bzip2 ca-certificates g++ gcc gfortran git gzip \
	libgl-dev libosmesa6-dev lsb-release patch python3 tar unzip xz-utils zstd

	- name: Set up Spack
	uses: spack/setup-spack@16b756799ede8b28951287f89bcd3fdb32ec1ca3 # v3.0.0
	with:
	spack_ref: v1.2.2
	packages_ref: d342cee94fb90fb020776c8bd8500b54a19c63cf
	# packages_ref: v2026.06.0

	# Pinned, like spack itself and its packages repo above: an unpinned
	# 'main' changes what CI builds with no commit here, which makes a
	# failure impossible to attribute. Bump deliberately.
	- name: Get FEniCS Spack packages
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	repository: fenics/spack-fenics
	ref: 55b14e0301eb64d4adc86c213046824f2b17e20c # main, 2026-08-15
	path: spack-fenics

	# Add FEniCS Spack packages on top of the upstream Spack packages
	- name: Add FEniCS Spack packages repo
	run: \|
	spack repo add --name fenics $GITHUB_WORKSPACE/spack-fenics/spack_repo/fenics

	- name: Get DOLFINx code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	path: dolfinx-src

	- name: Load dev branch environment variables
	run: cat dolfinx-src/.github/workflows/fenicsx-refs.env >> $GITHUB_ENV

	- name: Get UFL code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	repository: ${{ env.ufl_repository }}
	ref: ${{ env.ufl_ref }}
	path: ufl-src
	- name: Get FFCx code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	repository: ${{ env.ffcx_repository }}
	ref: ${{ env.ffcx_ref }}
	path: ffcx-src
	- name: Get Basix code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	repository: ${{ env.basix_repository }}
	ref: ${{ env.basix_ref }}
	path: basix-src

	- name: Record source commit SHAs
	run: \|
	echo "BASIX_SHA=$(git -C $GITHUB_WORKSPACE/basix-src rev-parse HEAD)" >> $GITHUB_ENV
	echo "UFL_SHA=$(git -C $GITHUB_WORKSPACE/ufl-src rev-parse HEAD)" >> $GITHUB_ENV
	echo "FFCX_SHA=$(git -C $GITHUB_WORKSPACE/ffcx-src rev-parse HEAD)" >> $GITHUB_ENV

	- name: Prepare Spack environment
	run: \|
	spack env create ci dolfinx-src/.github/workflows/spack-config/gh-actions-env-test.yml
	spack -e ci config add "packages:mpi:require:${{ matrix.mpi }}"
	for pkg in fenics-basix:basix py-fenics-basix:basix py-fenics-ufl:ufl \
	py-fenics-ffcx:ffcx fenics-ufcx:ffcx fenics-dolfinx:dolfinx py-fenics-dolfinx:dolfinx; do
	spack -e ci develop --no-clone --path $GITHUB_WORKSPACE/${pkg#:}-src ${pkg%%:}@main
	done

	# A single environment builds DOLFINx once for both the C++ and the
	# Python test steps. Separate environments resolve to an identical
	# runtime spec and differ only in build-time dependencies, so the
	# second build produces the same library at a different hash.
	- name: Build DOLFINx and test dependencies
	run: \|
	# Ensure build provenance by adding manually and providing git sha
	spack -e ci add fenics-basix@main commit=$BASIX_SHA build_type=Developer
	spack -e ci add py-fenics-basix@main commit=$BASIX_SHA build_type=Developer
	spack -e ci add py-fenics-ufl@main commit=$UFL_SHA
	spack -e ci add fenics-ufcx@main commit=$FFCX_SHA
	spack -e ci add py-fenics-ffcx@main commit=$FFCX_SHA
	# NOTE: adios2~sz drops the SZ2 compression dependency. 'sz'
	# installs its own unversioned libzstd.so, which interposes on the
	# real zstd once libdolfinx is loaded; a ZSTD_DCtx then gets
	# allocated in one zstd and freed in the other. This segfaults
	# Mesa's llvmpipe shader disk cache, so any PyVista render
	# following 'import dolfinx' crashes (see 'Run Python demos').
	spack -e ci add fenics-dolfinx@main+adios2${{ matrix.petsc && '+petsc+slepc' \|\| '~petsc~slepc+superlu-dist' }} commit=${{ github.sha }} build_type=Developer partitioners=parmetis \
	${{ matrix.petsc_dep }} ^adios2~libcatalyst~fortran~sz
	spack -e ci add py-fenics-dolfinx@main${{ matrix.petsc && '+petsc4py+slepc4py' \|\| '~petsc4py~slepc4py' }} commit=${{ github.sha }} build_type=Developer
	spack -e ci add py-gmsh ^gmsh~med~fltk+external ^opencascade~draw py-pyamg py-matplotlib \
	py-networkx \
	py-numba ^llvm~clang~lldb~lld~offload~libomptarget~polly targets=x86 \
	py-pytest py-pytest-xdist py-scipy \
	"py-pyvista ^[virtuals=gl] osmesa"
	spack -e ci add catch2 ninja
	spack -e ci concretize -j $(nproc)
	spack -e ci install -j $(nproc) --use-buildcache auto

	# Pushed before the test steps so the artifacts are cached even if a
	# later step fails. The index is rebuilt by the final push.
	#
	# Both pushes are skipped for pull requests from forks: those run
	# with a read-only GITHUB_TOKEN whatever 'permissions' asks for, so
	# the push would 401 as soon as it had anything to upload, failing a
	# contributor's job on a step that tests nothing. Reads are
	# anonymous, so their builds still take from the cache.
	- name: Push packages to cache
	if: >-
	${{ !cancelled() && (github.event_name != 'pull_request'
	\|\| github.event.pull_request.head.repo.full_name == github.repository) }}
	run: \|
	spack -e ci buildcache push --base-image ubuntu:26.04 --only dependencies \
	--with-build-dependencies --allow-missing local-buildcache

	- name: Build and run C++ unit tests (serial and MPI)
	shell: spack-bash {0}
	run: \|
	spack env activate ci
	spack load catch2 cmake ninja py-fenics-ffcx
	cmake -Werror=dev --warn-uninitialized -G Ninja -DCMAKE_BUILD_TYPE=Developer -B build/test/ -S dolfinx-src/cpp/test/
	cmake --build build/test
	cd build/test
	ctest --output-on-failure -R unittests_np_1
	ctest --output-on-failure -R unittests_np_3

	# Skipped without PETSc: several demo CMakeLists reference
	# PETSC_INCLUDE_DIRS unconditionally, which --warn-uninitialized
	# turns into an error, and the top-level demo CMakeLists adds every
	# subdirectory, so configuration fails as a whole.
	- name: Build and run C++ demos (serial and MPI)
	if: ${{ matrix.petsc }}
	shell: spack-bash {0}
	run: \|
	spack env activate ci
	spack load cmake ninja py-fenics-ffcx
	cmake -Werror=dev --warn-uninitialized -G Ninja -DCMAKE_BUILD_TYPE=Developer -B build/demo/ -S dolfinx-src/cpp/demo/
	cmake --build build/demo
	cd build/demo
	ctest --output-on-failure -R "demo_.*_np_1"
	ctest --output-on-failure -R "demo_.*_np_2"

	# Guard against a mis-specified variant silently building PETSc into
	# the PETSc-free leg, which would make it test nothing new.
	- name: Check PETSc configuration matches the matrix
	shell: spack-bash {0}
	run: \|
	spack env activate ci
	python -c "import dolfinx; assert dolfinx.has_petsc is ${{ matrix.petsc && 'True' \|\| 'False' }}, dolfinx.has_petsc"

	# mypy runs in ccpp.yml only in a build without PETSc, where it
	# excludes 'fem/petsc.py' and 'io/utils.py' because they need
	# petsc4py. Those modules are therefore type-checked nowhere. This
	# leg has petsc4py, gmsh and pyvista installed, so it checks them.
	#
	# The two disabled codes are the accepted limitation recorded in
	# AGENTS.md: petsc4py ships no stubs, so every signature touching it
	# degrades to 'Any'.
	#
	# One leg only: the result does not vary with the scalar type.
	- name: mypy checks (with petsc4py)
	if: ${{ matrix.petsc && matrix.mpi == 'openmpi' && matrix.scalar == 'float64' && matrix.index == 'int32' }}
	shell: spack-bash {0}
	run: \|
	spack env activate ci
	# mypy checks the sources; it is not a dependency of the build,
	# so keep it off the concretized DAG and out of the build cache.
	# The environment's python has no pip -- spack builds it without
	# one -- so install into a throwaway venv made with the
	# container's python, and point mypy at the environment's
	# interpreter with '--python-executable' so it resolves dolfinx,
	# petsc4py and the rest from there.
	apt-get install -y --no-install-recommends python3-venv
	python3 -m venv /tmp/mypy-venv
	/tmp/mypy-venv/bin/pip install --quiet mypy
	# Echo the version: this step is expected to report type errors,
	# so a broken install must not look the same as a working one.
	/tmp/mypy-venv/bin/mypy --version
	ENV_PYTHON=$(command -v python)
	MYPY="/tmp/mypy-venv/bin/mypy --python-executable $ENV_PYTHON"
	# No --disable-error-code: the other mypy invocations in CI pass
	# those because they check code they cannot import. This one has
	# petsc4py, gmsh and pyvista installed, so it runs against the
	# project config alone.
	cd dolfinx-src/python
	$MYPY --config-file pyproject.toml -p dolfinx
	$MYPY demo
	$MYPY test

	- name: Set default DOLFINx JIT options
	run: \|
	mkdir -p ~/.config/dolfinx
	echo '{ "cffi_extra_compile_args": ["-g0", "-O0" ] }' > ~/.config/dolfinx/dolfinx_jit_options.json

	- name: Run Python demos (serial and MPI)
	shell: spack-bash {0}
	run: \|
	spack env activate ci
	spack load gcc
	python -m pytest -n=auto -m serial --durations=10 dolfinx-src/python/demo/test.py
	python -m pytest -m mpi --num-proc=3 dolfinx-src/python/demo/test.py

	# NOTE: the marker expression is written '!matrix.petsc && X \|\| Y'
	# rather than 'matrix.petsc && Y \|\| X'. GitHub expressions treat the
	# empty string as false, so the latter form falls through to X on
	# every leg and silently skips the petsc4py tests everywhere.
	- name: Run Python unit tests (serial and MPI)
	shell: spack-bash {0}
	run: \|
	spack env activate ci
	spack load gcc
	python -m pytest -n=auto --durations=50 ${{ !matrix.petsc && '-m "not petsc4py"' \|\| '' }} dolfinx-src/python/test/unit/
	mpiexec -np 3 python -m pytest --durations=50 ${{ !matrix.petsc && '-m "not petsc4py"' \|\| '' }} dolfinx-src/python/test/unit/

	# TEMPORARY DIAGNOSTIC (remove once the crash stays away).
	# Creating a high-degree hexahedral H(curl) element segfaults with
	# 'rip' null on a varying subset of the matrix legs, always on an AMD
	# EPYC 9V74 runner. This step established where the fault is: the
	# same element builds fine on the same machine when LD_PRELOAD
	# supplies the reference BLAS, and valgrind reports no memory error
	# anywhere in the construction -- so the fault is in OpenBLAS, not in
	# basix. See the blas/lapack entry in the env config for the
	# mechanism, and for why the reference BLAS is now used instead.
	#
	# It stays until a run confirms the fix on an affected runner, since
	# only such a run can tell 'fixed' from 'did not land on an EPYC leg'.
	# Guarded on 'failure()', so it costs nothing on a green leg.
	- name: Diagnose segfaulting element creation
	if: failure()
	continue-on-error: true
	shell: spack-bash {0}
	run: \|
	apt-get install -y gdb valgrind libblas3 liblapack3
	spack env activate ci
	spack load gcc
	echo "== CPU =="
	lscpu \| grep -E "Model name\|^Flags" \| cut -c1-240
	cat > /tmp/probe_maps.py <<'EOF'
	import basix.ufl # noqa: F401
	libs = {ln.split()[-1] for ln in open("/proc/self/maps") if "blas" in ln.lower()}
	print(*sorted(libs), sep="\n")
	EOF
	echo "== BLAS libraries mapped alongside basix =="
	python /tmp/probe_maps.py \|\| true
	echo "== unresolved symbols in the basix extension =="
	ldd -r $(python -c "import basix._basixcpp as m; print(m.__file__)") 2>&1 \
	\| grep -iE "undefined\|not found" \| head -20 \|\| echo "none"
	# libbasix is what calls the BLAS; the extension above only wraps it.
	echo "== libbasix: BLAS linkage and unresolved symbols =="
	LIBBASIX=$(spack location -i fenics-basix)/lib/libbasix.so
	ldd "$LIBBASIX" \| grep -iE "blas\|lapack" \|\| echo "no BLAS in DT_NEEDED"
	ldd -r "$LIBBASIX" 2>&1 \| grep -iE "undefined\|not found" \| head -20 \|\| echo "none"
	cat > /tmp/repro.py <<'EOF'
	import basix.ufl
	basix.ufl.element("Nedelec 1st kind H(curl)", "hexahedron", 4)
	print("element created")
	EOF
	echo "== repro, with OpenBLAS core detection =="
	OPENBLAS_VERBOSE=2 python /tmp/repro.py \|\| true
	# If the reference BLAS survives what OpenBLAS does not, the fault
	# is in the BLAS; if both crash, the BLAS is a bystander.
	echo "== repro, against the reference BLAS =="
	LD_PRELOAD=/usr/lib/x86_64-linux-gnu/liblapack.so.3:/usr/lib/x86_64-linux-gnu/libblas.so.3 \
	python /tmp/repro.py \|\| true
	# Each BLAS entry prints its caller, so the last line before the
	# crash is the call site -- 'bt' cannot give it once rip is null.
	cat > /tmp/gdb.cmds <<'EOF'
	set auto-load safe-path /
	set disable-randomization off
	set pagination off
	set breakpoint pending on
	break dgemm_
	break dgesv_
	break dgetrf_
	break dgetri_
	break dsyevd_
	commands
	silent
	frame 1
	continue
	end
	run
	echo \n== CRASH ==\n
	bt
	x/512a $sp
	info registers
	EOF
	echo "== repro under gdb, tracing BLAS call sites =="
	gdb -q -batch -x /tmp/gdb.cmds --args python /tmp/repro.py 2>&1 \| tail -120 \|\| true
	# A jump to address zero leaves no return address on the stack, so
	# no unwinder can name the caller -- 'x/512a $sp' came back all
	# zeros. Valgrind tracks the call chain itself, so it reports the
	# jump ('Jump to the invalid address ... at 0x0: ??? by 0x...')
	# together with the frames that led there.
	echo "== repro under valgrind =="
	valgrind --error-limit=no --num-callers=30 --trace-children=yes \
	python /tmp/repro.py 2>&1 \| tail -60 \|\| true
	# Fallback, in case the one-liner above does not reproduce.
	echo "== affected tests serially under gdb =="
	gdb -q -batch -ex "set auto-load safe-path /" -ex "set disable-randomization off" \
	-ex "handle SIGPIPE nostop noprint pass" -ex run \
	-ex "bt" -ex "x/64a \$rsp" -ex "info registers" \
	--args python -m pytest -x -p no:cacheprovider -m "not petsc4py" \
	dolfinx-src/python/test/unit/fem/test_discrete_operators.py \|\| true

	# The test tools are environment roots, so '--only dependencies'
	# skips them and they need naming explicitly. Only the final push
	# rebuilds the index, which is the expensive part.
	- name: Push test packages to cache and update index
	if: >-
	${{ !cancelled() && (github.event_name != 'pull_request'
	\|\| github.event.pull_request.head.repo.full_name == github.repository) }}
	run: \|
	spack -e ci buildcache push --base-image ubuntu:26.04 --allow-missing --update-index local-buildcache \
	py-gmsh py-matplotlib py-networkx py-numba py-pytest py-pytest-xdist py-scipy py-pyvista

Navigation Menu

Uh oh!

Uh oh!

Fix the Spack CI segfault, and the test failures it was masking #3683

Workflow file

Fix the Spack CI segfault, and the test failures it was masking #3683

Uh oh!

Workflow file for this run