gstlal_inspiral_summary_page

#!/usr/bin/env python
#
# Copyright (C) 2011 Chad Hanna
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

## @file
# This program makes a summary web page of a gstlal inspiral analysis.

import sys, os, socket, re
import glob, math, shutil
import numpy
from glue import cbcwebpage
from glue import lal
from glue import segments
from optparse import OptionParser

from gstlal import inspiral_pipe
from pylal import git_version
__author__ = "Chad Hanna <channa@ligo.caltech.edu>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date


def parse_command_line():
    parser = OptionParser(version = git_version.verbose_msg, usage = "%prog [options] [file ...]", description = "%prog summary web page")
    parser.add_option("--webserver-dir", help = "Set the directory for the webserver.  Required.  Example /archive/home/channa/public_html/highmass_months_23-24_summary_page")
    parser.add_option("--open-box", action = "store_true", help = "Produce open box page")
    parser.add_option("--output-user-tag", action = "append", metavar = "name", help = "Set the basename for image search")
    parser.add_option("--title", default="gstlal_inspiral offline analysis")
    parser.add_option("--glob-path", help = "This is the path to search for images")

    opts, filenames = parser.parse_args()
    return opts, filenames

# PARSE THE COMMAND LINE
opts, files = parse_command_line()

# INITIALIZE THE PAGE OBJECT
page = cbcwebpage.cbcpage(title=opts.title)

# add external frames
for f in files:
    page.add_external_frame(f, f)

# WELCOME MESSAGE PAGE
page.front = "<big>Welcome</big>"

### Summary info. 
page.add_subpage("summary", "Summary information", "Summary information")
page.subpages["summary"].div("""
Summary information: This page provides information on the number of triggers, live time and horizon distance.
""")

# number of triggers
tab, name = cbcwebpage.wiki_table_parse(os.path.join(opts.glob_path, '%snum_trigs_table.txt' % opts.output_user_tag[0]))
if tab:
    page.subpages["summary"].add_table(tab[0], "Coinc events summary", "This table gives the number of coincident events")

# livetime
tab, name = cbcwebpage.wiki_table_parse(os.path.join(opts.glob_path, '%slive_time_table.txt' % opts.output_user_tag[0]))
if tab:
    page.subpages["summary"].add_table(tab[0], "Live time summary", "This table gives the summary of the live time")

imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*HORIZON*.png'))
page.subpages["summary"].add_table(imgtable, "Horizon Distance", "This is the distance at which you should see a 1.4 1.4 Msun binary at SNR 8. The first panel shows the time evolution of the horizon distance and the second panel shows the same information in histogram form. ")

### INJ PARAM PAGE ###
page.add_subpage("injection_params","Injection Parameters", "Injection Parameters")

page.subpages["injection_params"].div("""
<big><b>OVERVIEW:</b></big><br><br>
Injections are the addition of simulated signals into the gravitational wave strain data.  They are useful 
for diagnosing the pipeline and measuring the sensitivity.  Care is taken to ensure that the input 
distribution is sensible.  The remaining sections describe the input distributions for the injections.  
They are not necessarily what is analyzed since the analyzed segments are not used for determining the
injection sets (i.e. injections may be scheduled for times when a detector is not even functioning).
<br><br>The following plots are typically produced by injection waveform type.  Several injection runs
of the same waveform (with different random seeds) are often done.  Here all runs of the same waveform type
are shown together.
""")

page.subpages["injection_params"].add_section("m1_m2","Injected Parameters m1 and m2")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*6_sim_dist_m1_m2*.png'))
page.subpages["injection_params"].sections["m1_m2"].add_table(imgtable, "Injected mass1, mass2", "Above are the injected mass parameters for the simulations done.  Typically injections are done uniformally in component mass unless the waveform approximant cannot support certain mass ratios.")

page.subpages["injection_params"].add_section("time_dist","Injected Parameters time and distance")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*6_sim_dist_time_distance*.png'))
page.subpages["injection_params"].sections["time_dist"].add_table(imgtable, "Injection time and distance", "Unless doing a directed search injections are uniform in time.  Although the correct distance distribution would be uniform in volume typically logarithmic injections are done")

page.subpages["injection_params"].add_section("ra_dec","Injected Parameters RA and DEC")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*6_sim_dist_ra_dec*.png'))
page.subpages["injection_params"].sections["ra_dec"].add_table(imgtable, "Injection RA and DEC", "Unless doing a directed search injections are uniform on the sky.")

page.subpages["injection_params"].add_section("inc_pol","Injected Parameters inclination and polarization")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*6_sim_dist_inc_pol*.png'))
page.subpages["injection_params"].sections["inc_pol"].add_table(imgtable, "Injection Inclination and polarization", "Unless doing a directed search injections are uniform in sin(inclination) and polarizations.")

page.subpages["injection_params"].add_section("spin","Injected Parameters Z component spin")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*6_sim_dist_spin1z_spin2z*.png'))
page.subpages["injection_params"].sections["spin"].add_table(imgtable, "Injection spin 1z and spin2z", "The Z component spins of the injection set.  Often injections are done without spin, so don't be surprised if these are zero.")


### INJ ACC PAGE ###
page.add_subpage("injection_params_acc","Injection Parameters Accuracy", "Injection Accuracy")
page.subpages["injection_params_acc"].div("""
<big><b>OVERVIEW:</b></big><br><br>
The accuracy of recovered parameters aids at diagnosing the performance of the pipeline as well as
hinting at the possibility of measuring certain parameters.  Mass parameters for the low mass search 
are often recovered better than mass parameters for the high mass search.  
""")

page.subpages["injection_params_acc"].add_section("mchirp_param_acc","Mchirp Parameter Accuracy")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*2_mchirp_acc_frac_*.png'))
page.subpages["injection_params_acc"].sections["mchirp_param_acc"].add_table(imgtable, "Mchirp Accuracy", "Accuracy of chirp mass.")
page.subpages["injection_params_acc"].add_section("eta_param_acc","Eta Parameter Accuracy")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*2_eta_acc_frac_*.png'))
page.subpages["injection_params_acc"].sections["eta_param_acc"].add_table(imgtable, "Eta Accuracy", "Accuracy of eta")
page.subpages["injection_params_acc"].add_section("time_param_acc","Time Parameter Accuracy")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*2_t_acc_*.png'))
page.subpages["injection_params_acc"].sections["time_param_acc"].add_table(imgtable, "Time Accuracy", "Accuracy of time")
page.subpages["injection_params_acc"].add_section("deff_param_acc","Effective Distance Accuracy")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*2_deff_acc_frac*.png'))
page.subpages["injection_params_acc"].sections["deff_param_acc"].add_table(imgtable, "Effective Distance Accuracy", "Accuracy of effective distance")
page.subpages["injection_params_acc"].add_section("spin_param_acc","Effective Spin Accuracy")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*2_chi_acc*.png'))
page.subpages["injection_params_acc"].sections["spin_param_acc"].add_table(imgtable, "Effective Spin Accuracy", "Accuracy of mass-weighted spin")

### MISSED FOUND ###
page.add_subpage("missed_found","Missed Found", "Missed Found")
#page.subpages["missed_found"].add_section("overview","Missed found injections overview")
page.subpages["missed_found"].div("""
<big><b>OVERVIEW:</b></big><br><br>
Measuring the found and missed injections as a function of various parameters aids in diagnosing the 
pipeline as well as providing the expected sensitivity of the pipeline to real signals.  The plots in
this section show the missed and found injections as a for the various IFO times for coincident triggers.
We allow double coincident events so some categories can have multiple types of found injections (for
 example H1L1 and H1H2L1 triggers in H1H2L1 time).  Because of ambiguity concerning the time of an 
injection and the injection window it is occasionally possible to find an injection in more detectors
than what the "time" refers to.  For example, an injection's geocentric end time might be in H1L1 time
but that might occur near a boundary where H2 was also on.  Thus one could find an H1L1 injection in
H1H2L1 time.
""")

page.subpages["missed_found"].add_section("found_missed_mchirp","Found / Missed Chirp Mass")
page.subpages["missed_found"].add_section("found_missed_mtotal","Found / Missed Total Mass")
page.subpages["missed_found"].add_section("found_missed_time","Found / Missed Time")
page.subpages["missed_found"].add_section("found_missed_chi","Found / Missed Spin")

for tag in opts.output_user_tag:
    tab,name = cbcwebpage.wiki_table_parse(os.path.join(opts.glob_path, '%sinjection_summary.txt' % tag))
    page.subpages["missed_found"].add_table(tab[0], "Injection Summary Table: %s" % tag, "Summary of missed and found injections broken up by detector time")

    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*1_d_vs_mchirp_*.png' % tag))
    page.subpages["missed_found"].sections["found_missed_mchirp"].add_table(imgtable, "Missed Found Distance vs Mchirp: %s" % tag, "Distance of found and missed injections as a function of chirp mass")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*1_deff_vs_mchirp_*.png' % tag))
    page.subpages["missed_found"].sections["found_missed_mchirp"].add_table(imgtable, "Missed Found Effective Distance vs Mchirp: %s" % tag, "Effective distance of found and missed injections as a function of chirp mass")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*1_chirpdist_vs_mchirp_*.png' % tag))
    page.subpages["missed_found"].sections["found_missed_mchirp"].add_table(imgtable, "Missed Found Chirp Distance vs Mchirp: %s" % tag, "Chirp distance of found and missed injections as a function of chirp mass.  For low mass systems the chirp mass scales out of the expected amplitude.  The chirp distance utilizes this and rescales the distance to be appropriate for a NS-NS binary.  At low total mass the missed/found barrier should be flat on this plot.")

    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*1_deff_vs_mtotal_*.png' % tag))
    page.subpages["missed_found"].sections["found_missed_mtotal"].add_table(imgtable, "Missed Found Effective Distance vs M total: %s" % tag, "Effective distance of found and missed injections as a function of total mass")

    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*1_deff_vs_t_*.png' % tag))
    page.subpages["missed_found"].sections["found_missed_time"].add_table(imgtable, "Missed Found vs Time: %s" % tag, "Effective distance of found and missed injections as a function of time")

    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*1_deff_vs_chi_*.png' % tag))
    page.subpages["missed_found"].sections["found_missed_chi"].add_table(imgtable, "Missed Found vs Spin: %s" % tag, "Effective distance of found and missed injections as a function of mass-weighted spin")

# SEARCH SENSITIVITY PLOTS #
page.add_subpage("search_sensitivity","Search Sensitivity", "Search Sensitivity")
page.subpages["search_sensitivity"].div("""
<big><b>OVERVIEW:</b></big><br><br>
This page shows plots that characterize the overall sensitivity of the search. The detection rate is directly proportional to the observed sensitive volume, which is a function of the significance threshold used to identify candidates. These plots are useful for determining how sensitive the search was to the input injection population. Since the search sensitivity is a strong function of the intrinsic parameters of the target source, we break down the sensitive into bins of these parameters. Here we show the sensitivity as a function of total mass and component mass.
""")
page.subpages["search_sensitivity"].add_section("chirp_mass","Binned by chirp mass")
page.subpages["search_sensitivity"].add_section("total_mass","Binned by total mass")
page.subpages["search_sensitivity"].add_section("mass1_mass2","Binned by mass1-mass2")
page.subpages["search_sensitivity"].add_section("aligned_spin","Binned by aligned spin")
page.subpages["search_sensitivity"].add_section("mass_ratio","Binned by mass ratio")

for tag in opts.output_user_tag:
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*BINNED_BY_CHIRP_MASS*.png' % tag))
    page.subpages["search_sensitivity"].sections["chirp_mass"].add_table(imgtable, "Sensitive volume (V*T) and mean sensitive distance: %s" % tag, "Binned by chirp mass")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*BINNED_BY_TOTAL_MASS*.png' % tag))
    page.subpages["search_sensitivity"].sections["total_mass"].add_table(imgtable, "Sensitive volume (V*T) and mean sensitive distance: %s" % tag, "Binned by total mass")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*BINNED_BY_MASS1_MASS2*.png' % tag))
    page.subpages["search_sensitivity"].sections["mass1_mass2"].add_table(imgtable, "Sensitive volume (V*T) and mean sensitive distance: %s" % tag, "Binned by component mass")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*BINNED_BY_ALIGNED_SPIN*.png' % tag))
    page.subpages["search_sensitivity"].sections["aligned_spin"].add_table(imgtable, "Sensitive volume (V*T) and mean sensitive distance: %s" % tag, "Binned by aligned spin")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*%s*BINNED_BY_MASS_RATIO*.png' % tag))
    page.subpages["search_sensitivity"].sections["mass_ratio"].add_table(imgtable, "Sensitive volume (V*T) and mean sensitive distance: %s" % tag, "Binned by mass ratio")


# BACKGROUND#
page.add_subpage("background","Background", "Background")
page.subpages["background"].div("""
<big><b>OVERVIEW:</b></big><br><br>
This page shows the ingredients that go into ranking events and determining the False alarm probability.
""")
page.subpages["background"].add_section("snrchi","SNR and chi-squared")
page.subpages["background"].sections["snrchi"].div("This section gives the SNR / Chisq background statistics.")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*SNRCHI2*.png'))
page.subpages["background"].sections["snrchi"].add_table(imgtable, "","")

page.subpages["background"].add_section("snrsnr","SNR ratios")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*SNR_PDF*.png'))
page.subpages["background"].sections["snrsnr"].add_table(imgtable, "","")

page.subpages["background"].add_section("rates","Rates")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*RATES*.png'))
page.subpages["background"].sections["rates"].add_table(imgtable, "","")

page.subpages["background"].add_section("rankingstat","Ranking Statistic Models")
page.subpages["background"].sections["rankingstat"].div("This section shows the ranking statistic (log likelihood ratio) PDFs and CCDFs for the noise and signal models.")
imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*LIKELIHOOD_RATIO_*.png'))
page.subpages["background"].sections["rankingstat"].add_table(imgtable, "","")


#
# CLOSED BOX
#

if not opts.open_box:
    
    ### CHISQ ###
    page.add_subpage("chisq","Chi-squared", "Chi-squared")
    page.subpages["chisq"].div("""
<big><b>OVERVIEW:</b></big><br><br>
The chi-squared test checks that the snr accumulated matches expectation. 
    """)
    page.subpages["chisq"].add_section("chisq","Chi-squared vs SNR")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*3_chi2_vs_rho_*closedbox*.png'))
    page.subpages["chisq"].sections["chisq"].add_table(imgtable, "Chi-squared Vs SNR", "Chi-squared vs snr for single detectors after coincidence.  Blue points are full data zero lag, red are software injections and black are time slides.")

    page.subpages["chisq"].add_section("snr","Signal-to-Noise Ratio")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*4_rho_*_vs_*closedbox*.png'))
    page.subpages["chisq"].sections["snr"].add_table(imgtable, "Signal-to-Noise Ratio", "Comparison of SNR in pairs of detectors.")

    page.subpages["chisq"].add_section("deff","Effective Distance")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*4_deff_*_vs_*closedbox*.png'))
    page.subpages["chisq"].sections["deff"].add_table(imgtable, "Effective Distance", "Comparison of effective distance in pairs of detectors.")

    ### Money Plots ###
    page.add_subpage("money","Money Plots", "Money Plots")
    page.subpages["money"].div("""
<big><b>OVERVIEW:</b></big><br><br>
This section provides the detection statistic plots and a summary of the loudest event.
""")
    page.subpages["money"].add_section("ifar","Rate vs. IFAR Threshold")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*5_count_vs_ifar*closedbox*.png'))
    page.subpages["money"].sections["ifar"].add_table(imgtable, "Rate vs. IFAR Threshold", "Comparison of observed zero-lag event rate to event rate expected from background as a function of inverse false-alarm rate (IFAR) threshold.")
    page.subpages["money"].add_section("table","Loudest Table")
    tab,name = cbcwebpage.wiki_table_parse(os.path.join(opts.glob_path, '%sbgsummary_table.txt' % opts.output_user_tag[0]))
    page.subpages["money"].sections["table"].add_table(tab[0], "Closed Box Summary Table", "Loudest Events")

#
# OPEN BOX
#

else:
    print >>sys.stderr, "WARNING: OPENING THE BOX"
    
    ### CHISQ ###
    page.add_subpage("chisq","Chi-squared", "Chi-squared")
    page.subpages["chisq"].div("""
<big><b>OVERVIEW:</b></big><br><br>
The chi-squared test checks that the snr accumulated matches expectation. 
FIXME
    """)
    page.subpages["chisq"].add_section("chisq","Chi-squared vs SNR")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*3_chi2_vs_rho_*openbox*.png'))
    page.subpages["chisq"].sections["chisq"].add_table(imgtable, "Chi-squared Vs SNR", "Chi-squared vs snr for single detectors after coincidence.  Blue points are full data zero lag, red are software injections and black are time slides.")

    page.subpages["chisq"].add_section("snr","Signal-to-Noise Ratio")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*4_rho_*_vs_*openbox*.png'))
    page.subpages["chisq"].sections["snr"].add_table(imgtable, "Signal-to-Noise Ratio", "Comparison of SNR in pairs of detectors.")

    page.subpages["chisq"].add_section("deff","Effective Distance")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*4_deff_*_vs_*openbox*.png'))
    page.subpages["chisq"].sections["deff"].add_table(imgtable, "Effective Distance", "Comparison of effective distance in pairs of detectors.")

    ### Money Plots ###
    page.add_subpage("money","Money Plots", "Money Plots")
    page.subpages["money"].div("""
<big><b>OVERVIEW:</b></big><br><br>
This section provides the detection statistic plots and a summary of the loudest event.
""")
    page.subpages["money"].add_section("ifar","Rate vs. IFAR Threshold")
    imgtable = cbcwebpage.image_glob(os.path.join(opts.glob_path, '*5_count_vs_ifar*openbox*.png'))
    page.subpages["money"].sections["ifar"].add_table(imgtable, "Rate vs. IFAR Threshold", "Comparison of observed zero-lag event rate to event rate expected from background as a function of inverse false-alarm rate (IFAR) threshold.")
    page.subpages["money"].add_section("table","Loudest Table")
    tab,name = cbcwebpage.wiki_table_parse(os.path.join(opts.glob_path, '%ssummary_table.txt' % opts.output_user_tag[0]))
    page.subpages["money"].sections["table"].add_table(tab[0], "Summary Table", "Loudest Events")

    ### UPPER LIMIT PLOTS ###
    #page.add_subpage("ul","Upper Limit Plots", "Upper Limit Plots")
    #page.subpages["ul"].div("""
#<big><b>OVERVIEW:</b></big><br><br>
#This section describes the upperlimit calculation which is a 90% confidence upper limit on the rate of mergers with units mergers/Mpc^3/year as a function of mass.  Various 
#quantities go into this calculation.  They are shown here. 
#""")

    #range_summs = glob.glob(base_name+'-*_range_summary.txt')
    #for rs in range_summs:
    #   tag = rs.replace('_',' ').replace('.txt','')
    #   page.subpages["ul"].add_section(tag,rs)
    #   tab,name = cbcwebpage.wiki_table_parse(rs)
    #   page.subpages["ul"].sections[tag].add_table(tab[0], "Range Summary", "Range summary")

    #page.subpages["ul"].add_section("vt","Volume x time")
    #imgtable = cbcwebpage.image_glob(base_name+'-*_volume_time.png')
    #page.subpages["ul"].sections["vt"].add_table(imgtable, "Volume x time", "mass1 mass2 volume x time")

    #page.subpages["ul"].add_section("evt","Error on Volume x time")
    #imgtable = cbcwebpage.image_glob(base_name+'-*_fractional_error.png')
    #page.subpages["ul"].sections["evt"].add_table(imgtable, "Error on Volume x time", "Error on mass1 mass2 volume x time")

    #page.subpages["ul"].add_section("l","Lambda")
    #imgtable = cbcwebpage.image_glob(base_name+'-*_lambda.png')
    #page.subpages["ul"].sections["l"].add_table(imgtable, "Lambda", "likelihood fg/bg @ loudest event")

    #page.subpages["ul"].add_section("p","Posterior")
    #imgtable = cbcwebpage.image_glob(base_name+'-*_posterior.png')
    #page.subpages["ul"].sections["p"].add_table(imgtable, "Posterior", "Poseterior on the rate")

    #page.subpages["ul"].add_section("ul","90% Upper limit")
    #imgtable = cbcwebpage.image_glob(base_name+'-*_upper_limit.png')
    #page.subpages["ul"].sections["ul"].add_table(imgtable, "90% Upper limit", "90% upper limit")

    #page.subpages["ul"].add_section("cul","Combined 90% Upper limit")
    #imgtable = [[cbcwebpage._imagelinkcpy(base_name+'upper_limit.png'), cbcwebpage._imagelinkcpy(base_name+'posterior.png')]]
    #page.subpages["ul"].sections["cul"].add_table(imgtable, "Combined 90% Upper limit", "Combined 90% upper limit")


if opts.open_box:
    page.write("%s_openbox" % opts.output_user_tag[0])
else:
    page.write("%s_closebox" % opts.output_user_tag[0])
    
# copy the output
if opts.webserver_dir:
    for f in page.external_frames:
        shutil.copy(f[0], opts.webserver_dir)
    for f in page.fnames: 
        shutil.copy(f,opts.webserver_dir)
    #FIXME not the best way to do this, I should explicitely save file names
    try: os.mkdir(opts.webserver_dir+'/Images')
    except: pass
    for f in glob.glob('Images/*.png'):
        shutil.copy(f,opts.webserver_dir+'/Images')