Skip to content

Add redshift QA scripts #2521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add redshift QA scripts #2521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a3d50e
add redshift QA scripts
rongpu Jul 25, 2025
bae1528
use consistent lrg/elg mask files
rongpu Jul 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions py/desispec/redshiftqa/desi_per_fiber_qa_commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Compute the per-fiber redshift success rates and save them to disk
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_stats.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ -o per_fiber_qa_stats.fits

# Make per-fiber per-tracer redshift QA plots
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_plots.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ --tracer LRG --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_plots.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ --tracer ELG_LOP --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_plots.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ --tracer ELG_VLO --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_plots.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ --tracer QSO --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_plots.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ --tracer BGS_BRIGHT --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa
salloc -N 1 -C cpu -t 04:00:00 -q interactive ./desi_per_fiber_qa_plots.py -i /global/cfs/cdirs/desicollab/users/rongpu/tmp/zcatalog/v0.4/main/ --tracer BGS_FAINT --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa

# Create the redshift QA HTML files
./desi_per_fiber_qa_html.py --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa

146 changes: 146 additions & 0 deletions py/desispec/redshiftqa/desi_per_fiber_qa_html.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#!/usr/bin/env python

# Create the redshift QA HTML files
# Usage:
# ./desi_per_fiber_qa_html.py --stats per_fiber_qa_stats.fits -o /global/cfs/cdirs/desicollab/users/rongpu/redshift_qa/new/loa

from __future__ import division, print_function
import sys, os, glob, time, warnings, gc
import matplotlib.pyplot as plt
import numpy as np
from astropy.table import Table, vstack, hstack
import fitsio
from astropy.io import fits

import argparse

pvalue_threshold = 1e-4

parser = argparse.ArgumentParser()
parser.add_argument('--stats', type=str, help="per-fiber stats path", required=True)
parser.add_argument('-o', '--output', type=str, help="output directory path", required=True)
args = parser.parse_args()

stats_fn = args.stats
output_dir = args.output
html_dir = os.path.join(output_dir, 'html')

stats = Table(fitsio.read(stats_fn))

if not os.path.isdir(html_dir):
os.makedirs(html_dir)

f = open(os.path.join(output_dir, 'fiber_directory.html'), "w")
f.write('<html>\n')
f.write('<table>\n')
for fiber in np.arange(5000):
f.write('<td><a href=html/fiber_{}.html>FIBER_{}</a></td>\n'.format(fiber, fiber))
f.write('</tr>\n')
f.write('</table>\n')
f.close()

for fiber in np.arange(5000):

mask = stats['FIBER']==fiber
index = np.where(mask)[0][0]

f = open(os.path.join(html_dir, 'fiber_{}.html'.format(fiber)), "w")
f.write('<html>\n')

f.write('<style>\ntable, th, td {\n border:1px solid black;\n}\n</style>\n')

f.write('<p>')
if fiber==0:
f.write('<a href="fiber_1.html">Next Fiber</a>')
elif fiber==4999:
f.write('<a href="fiber_0.html">Previous Fiber</a>')
else:
f.write('<a href="fiber_{}.html">Previous Fiber</a> &nbsp; <a href="fiber_{}.html">Next Fiber</a>'.format(fiber-1, fiber+1))
f.write('</p>')

################################## summary table ##################################
f.write('<table>\n')

f.write('<tr>\n')
f.write('<th></th>\n')
for tracer in ['BGS_BRIGHT', 'LRG', 'ELG_LOP', 'QSO', 'ELG_VLO', 'BGS_FAINT']:
for apply_good_z_cut in [False, True]:
if apply_good_z_cut is False:
f.write('<th>{} (all)</th>\n'.format(tracer))
else:
f.write('<th>{} (good z)</th>\n'.format(tracer))
f.write('</tr>\n')

f.write('<tr>\n')
f.write('<th>N(z) p-value</th>\n')
for tracer in ['BGS_BRIGHT', 'LRG', 'ELG_LOP', 'QSO', 'ELG_VLO', 'BGS_FAINT']:
for apply_good_z_cut in [False, True]:
if apply_good_z_cut is False:
pvalue_col = tracer.lower()+'_ks_pvalue_allz'
else:
pvalue_col = tracer.lower()+'_ks_pvalue_goodz'
pvalue = stats[pvalue_col][index]
if pvalue==-99:
f.write('<td>N/A</td>\n')
elif pvalue<pvalue_threshold:
f.write('<th><p style="color:red;">{:.4g}</p></th>\n'.format(pvalue))
else:
f.write('<td>{:.4g}</td>\n'.format(pvalue))
f.write('</tr>\n')

f.write('<tr>\n')
f.write('<th>fiber failure rate</th>\n')
for tracer in ['BGS_BRIGHT', 'LRG', 'ELG_LOP', 'QSO', 'ELG_VLO', 'BGS_FAINT']:
n_tot = stats[tracer.lower()+'_n_tot'][index]
n_fail = stats[tracer.lower()+'_n_fail'][index]
frac_fail = stats[tracer.lower()+'_frac_fail'][index]
frac_fail_err = stats[tracer.lower()+'_frac_fail_err'][index]
if n_tot==-99:
f.write('<td colspan="2">N/A</td>\n')
else:
f.write('<td colspan="2">{:.1f} &pm; {:.1f}% ({}/{})</td>\n'.format(frac_fail*100, frac_fail_err*100, n_fail, n_tot))
f.write('</tr>\n')

f.write('<tr>\n')
f.write('<th>overall failure rate</th>\n')
for tracer in ['BGS_BRIGHT', 'LRG', 'ELG_LOP', 'QSO', 'ELG_VLO', 'BGS_FAINT']:
mask = stats[tracer.lower()+'_n_tot']!=-99
if np.sum(mask)==0:
n_tot = -99
else:
n_tot = np.sum(stats[tracer.lower()+'_n_tot'][mask])
n_fail = np.sum(stats[tracer.lower()+'_n_fail'][mask])
frac_fail = n_fail/n_tot
if n_tot==-99:
f.write('<td colspan="2">N/A</td>\n')
else:
f.write('<td colspan="2">{:.1f}% ({}/{})</td>\n'.format(frac_fail*100, n_fail, n_tot))
f.write('</tr>\n')

f.write('</table>\n')

################################## QA plots ##################################

f.write('<table>\n')
for tracer in ['BGS_BRIGHT', 'LRG', 'ELG_LOP', 'QSO', 'ELG_VLO', 'BGS_FAINT']:

f.write('<th></th>\n')
f.write('<th>{}</th>\n'.format(tracer))
f.write('<tr>\n')

for apply_good_z_cut in [False, True]:

if apply_good_z_cut:
f.write('<td>Good z</td>\n')
png_dir = '{}_goodz'.format(tracer.lower())
else:
f.write('<td>All</td>\n')
png_dir = '{}_allz'.format(tracer.lower())

image_fn = os.path.join(png_dir, 'fiber_{}_{}.png'.format(fiber, os.path.basename(png_dir)))

f.write('<td><a href=\'../png/{}\'><img src=\'../png/{}\' width=\'1200\'></a></td>\n'.format(image_fn, image_fn))
f.write('</tr>\n')

f.write('</table>\n')
f.close()
Loading
Loading