@@ -227,95 +227,6 @@ jobs:
227227 python -m pytest -n=auto --durations=50 ${{ !matrix.petsc && '-m "not petsc4py"' || '' }} dolfinx-src/python/test/unit/
228228 mpiexec -np 3 python -m pytest --durations=50 ${{ !matrix.petsc && '-m "not petsc4py"' || '' }} dolfinx-src/python/test/unit/
229229
230- # TEMPORARY DIAGNOSTIC (remove once the crash is understood).
231- # Creating a high-degree hexahedral H(curl) element segfaults with
232- # 'rip' null on a varying subset of the matrix legs. Pinning the
233- # OpenBLAS kernel set ('openblas~dynamic_dispatch') did not help: the
234- # crash reproduced on an AMD EPYC 9V74 runner with a non-dispatching
235- # OpenBLAS whose compiled-in core ('Core: Zen') matched the CPU, and
236- # under OPENBLAS_CORETYPE and LD_BIND_NOW alike. So this now asks the
237- # question that ordering settles: whether the faulting call is a BLAS
238- # call at all. It breaks on the BLAS entry points to print each call
239- # site, and re-runs against the reference BLAS.
240- # Guarded on 'failure()', so it costs nothing on a green leg.
241- - name : Diagnose segfaulting element creation
242- if : failure()
243- continue-on-error : true
244- shell : spack-bash {0}
245- run : |
246- apt-get install -y gdb valgrind libblas3 liblapack3
247- spack env activate ci
248- spack load gcc
249- echo "== CPU =="
250- lscpu | grep -E "Model name|^Flags" | cut -c1-240
251- cat > /tmp/probe_maps.py <<'EOF'
252- import basix.ufl # noqa: F401
253- libs = {ln.split()[-1] for ln in open("/proc/self/maps") if "blas" in ln.lower()}
254- print(*sorted(libs), sep="\n")
255- EOF
256- echo "== BLAS libraries mapped alongside basix =="
257- python /tmp/probe_maps.py || true
258- echo "== unresolved symbols in the basix extension =="
259- ldd -r $(python -c "import basix._basixcpp as m; print(m.__file__)") 2>&1 \
260- | grep -iE "undefined|not found" | head -20 || echo "none"
261- # libbasix is what calls the BLAS; the extension above only wraps it.
262- echo "== libbasix: BLAS linkage and unresolved symbols =="
263- LIBBASIX=$(spack location -i fenics-basix)/lib/libbasix.so
264- ldd "$LIBBASIX" | grep -iE "blas|lapack" || echo "no BLAS in DT_NEEDED"
265- ldd -r "$LIBBASIX" 2>&1 | grep -iE "undefined|not found" | head -20 || echo "none"
266- cat > /tmp/repro.py <<'EOF'
267- import basix.ufl
268- basix.ufl.element("Nedelec 1st kind H(curl)", "hexahedron", 4)
269- print("element created")
270- EOF
271- echo "== repro, with OpenBLAS core detection =="
272- OPENBLAS_VERBOSE=2 python /tmp/repro.py || true
273- # If the reference BLAS survives what OpenBLAS does not, the fault
274- # is in the BLAS; if both crash, the BLAS is a bystander.
275- echo "== repro, against the reference BLAS =="
276- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/liblapack.so.3:/usr/lib/x86_64-linux-gnu/libblas.so.3 \
277- python /tmp/repro.py || true
278- # Each BLAS entry prints its caller, so the last line before the
279- # crash is the call site -- 'bt' cannot give it once rip is null.
280- cat > /tmp/gdb.cmds <<'EOF'
281- set auto-load safe-path /
282- set disable-randomization off
283- set pagination off
284- set breakpoint pending on
285- break dgemm_
286- break dgesv_
287- break dgetrf_
288- break dgetri_
289- break dsyevd_
290- commands
291- silent
292- frame 1
293- continue
294- end
295- run
296- echo \n== CRASH ==\n
297- bt
298- x/512a $sp
299- info registers
300- EOF
301- echo "== repro under gdb, tracing BLAS call sites =="
302- gdb -q -batch -x /tmp/gdb.cmds --args python /tmp/repro.py 2>&1 | tail -120 || true
303- # A jump to address zero leaves no return address on the stack, so
304- # no unwinder can name the caller -- 'x/512a $sp' came back all
305- # zeros. Valgrind tracks the call chain itself, so it reports the
306- # jump ('Jump to the invalid address ... at 0x0: ??? by 0x...')
307- # together with the frames that led there.
308- echo "== repro under valgrind =="
309- valgrind --error-limit=no --num-callers=30 --trace-children=yes \
310- python /tmp/repro.py 2>&1 | tail -60 || true
311- # Fallback, in case the one-liner above does not reproduce.
312- echo "== affected tests serially under gdb =="
313- gdb -q -batch -ex "set auto-load safe-path /" -ex "set disable-randomization off" \
314- -ex "handle SIGPIPE nostop noprint pass" -ex run \
315- -ex "bt" -ex "x/64a \$rsp" -ex "info registers" \
316- --args python -m pytest -x -p no:cacheprovider -m "not petsc4py" \
317- dolfinx-src/python/test/unit/fem/test_discrete_operators.py || true
318-
319230 # The test tools are environment roots, so '--only dependencies'
320231 # skips them and they need naming explicitly. Only the final push
321232 # rebuilds the index, which is the expensive part.
0 commit comments