1
2
3 import glob
4 import os
5 import sys
6 import subprocess
7 import shutil
8 import optparse
9 import urllib
10 import ConfigParser
11 import warnings
12 warnings.simplefilter('ignore',DeprecationWarning)
13 warnings.simplefilter('ignore',UserWarning)
14
15 itertools = __import__("itertools")
16
17 import numpy
18 import matplotlib; matplotlib.use('Agg')
19 import pylab as plt
20
21 from glue.ligolw import ligolw
22 from glue.ligolw import utils
23 from glue.ligolw import table
24 from glue.ligolw import lsctables
25 from glue import lal
26 from glue import segments
27 from glue import segmentsUtils
28 from glue import pipeline
29 from glue import iterutils
30
31
32
33
34
36 """
37 Parses the command line arguments.
38 """
39 parser = optparse.OptionParser()
40
41 parser.add_option("--config-file", help="Specifies the configuration file to use.")
42 parser.add_option("--log-file", help="Specifies the log file for any outputs.",default = "default.log")
43 parser.add_option("--grb-file", help="Full path to the GRB xml file.")
44
45 parser.add_option("--time", help="GPS time of the GRB trigger.")
46 parser.add_option("--ra", help="Right ascension of the GRB (degree).")
47 parser.add_option("--dec", help="Declination of the GRB (degree).")
48 parser.add_option("--name", help="Name of the GRB (e.g. 100122A).")
49
50 parser.add_option("--offset", help="Length of time to check in both directions [s].",type=int,default = 2500)
51 parser.add_option("--useold", action="store_true", default=False, help="If this option is specified, the old old segment files will be used. Otherwise they will be queries freshly.")
52 parser.add_option("--extend", action="store_true", default=False, help="Adds quanta of adjacent segments of data with duration of segment-duration/2.")
53
54 parser.add_option("--make-plots", action = "store_true", default = False, help = "If this option is specified, segment plots will be created.")
55 parser.add_option("--make-xml", action = "store_true", default = False, help = "If this option is specified, a xml file will be created.")
56
57 options, arguments = parser.parse_args()
58
59
60 if not options.config_file:
61 raise ValueError, "Must enter a configuration file to use."
62 if not options.grb_file and (not options.time or not options.name):
63 raise ValueError, "Either a valid GRB xml file must be specified or the GPS time and name of the GRB!"
64 if options.make_xml:
65 if not options.ra or not options.dec:
66 raise ValueError, "The right ascension (--ra) and the declination (--dec) is required to be specified, because --make-xml was set."
67 if options.grb_file and options.make_xml:
68 print "Warning! It does not make sense to specfy the xml file as input and as output. Therefore, the make-xml flag will be set to False"
69 options.make_xml = False
70
71 return options, arguments
72
73 -def plot_segments(segdict, onsource, offsource, centertime, plot_window, output_filename, tag):
74 """
75 Creates time series plots of segments for each IFO.
76 """
77
78 color_code = {'H1':'r', 'H2':'b', 'L1':'g', 'V1':'m', 'G1':'k'}
79
80
81 fig = plt.figure()
82 ax = fig.add_subplot(111)
83 ax.set_xlabel("time (s)")
84 ax.set_ylabel("IFO")
85 window = None
86
87
88 _time_transform = lambda t: t - centertime
89
90
91 ifolist = segdict.keys()
92 ifolist.sort()
93 for row, ifo in enumerate(ifolist):
94 color = color_code[ifo]
95 for seg in segdict[ifo]:
96 a = _time_transform(seg[0])
97 b = _time_transform(seg[1])
98 ax.fill([a, b, b, a, a], [row, row, row+1, row+1, row], color)
99
100
101 if offsource:
102 c = _time_transform(plot_window[0])
103 d = _time_transform(plot_window[1])
104 window = segments.segment((c, d))
105
106
107 if abs(plot_window) > 0:
108 a = _time_transform(offsource[0])
109 b = _time_transform(offsource[1])
110 ax.axvline(a, color='k', linewidth=2)
111 ax.axvline(b, color='k', linewidth=2)
112
113
114 ax.axvline(_time_transform(onsource[0]), color='k', linestyle='--')
115 ax.axvline(_time_transform(onsource[1]), color='k', linestyle='--')
116
117
118 ticks = plt.arange(len(ifolist)) + 0.5
119 ax.set_yticks(ticks)
120 ax.set_yticklabels(ifolist)
121
122
123 if window is not None:
124 ax.set_xlim(window)
125 ax.set_ylim((0, len(ifolist)))
126
127
128 ax.set_title('Segments for GRB '+tag)
129 fig.savefig(output_filename)
130 plt.close(fig)
131
133 '''
134 Searches +/- offset from GRB time to download the latest segment lists then extracts times and puts them into a txt file.
135 '''
136 args = {'grb_name' : grb_name,
137 'query_start' : query_start,
138 'query_end' : query_end,
139 'ifo' : ifo,
140 'segmentName' : segmentName}
141 cmd = "ligolw_segment_query --database --query-segments --include-segments '{segmentName}' --gps-start-time {query_start} --gps-end-time {query_end} > ./segments{ifo}_grb{grb_name}.xml".format(**args)
142 print '>>',cmd
143 print
144 process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
145 output,err = process.communicate()
146
147
148 try:
149 doc = utils.load_filename("segments{ifo}_grb{grb_name}.xml".format(**args), contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler))
150 except:
151 raise IOError, "Error reading file: segments{ifo}_grb{grb_name}.xml".format(**args)
152
153
154 segs = table.get_table(doc, "segment")
155 seglist = segments.segmentlist(segments.segment(s.start_time, s.end_time) for s in segs)
156 segmentsUtils.tosegwizard(file("{ifo}-science_grb{grb_name}.txt".format(**args),'w'),seglist,header = True)
157
158 print ">> %s segments +/-%ds from %ds found:"%(ifo,offset,grb_time)
159 for s in segs:
160 print "Start:",s.start_time,"End:",s.end_time,"Duration:",s.end_time-s.start_time
161 print
162
163 return
164
165 -def exttrig_dataquery(grb_name, grb_time, grb_ra, grb_dec, offset, config_file, extend=False, useold=False, make_plots=False, make_xml=False):
166 '''
167 Finds science time of all available IFOs.
168 '''
169
170
171
172
173
174 cp = ConfigParser.ConfigParser()
175 cp.read(config_file)
176
177
178 basic_ifolist = ifolist = ['H1','H2','L1','V1']
179 catlist = [1,2,3]
180 sensitivity_dict = {"H1": 1, "L1": 2, "H2": 3, "V1": 4, "G1": 5}
181
182
183 pad_data = int(cp.get('data','pad-data'))
184 if cp.has_option('data','segment-duration'):
185 blockDuration = segmentDuration = psdDuration = int(cp.get('data','segment-duration'))
186 elif cp.has_option('data','segment-length'):
187 blockDuration = segmentDuration = psdDuration = int(cp.get('data','segment-length')) /int(cp.get('data','sample-rate'))
188 else:
189 raise ValueError, "EXIT: Cannot find segment-duration in [data] section of configuration file!"
190
191
192 if cp.has_option('data','sample-rate'):
193 sampleRate = int(cp.get('data', 'sample-rate'))
194 print ">> Sample rate has been set to: %d"%sampleRate
195 print
196 else:
197 print ">> ERROR: Need to specify sample-rate in [data] section of configuration file in order to calculate inputs for downstream processes."
198 sys.exit()
199
200
201 if not extend:
202 if cp.has_option('data','block-duration'):
203 blockDuration = int(cp.get('data','block-duration'))
204 elif cp.has_option('data','segment-length'):
205 s_length = int(cp.get('data', 'segment-length'))
206 s_num = int(cp.get('data', 'number-of-segments'))
207 s_rate = int(cp.get('data', 'sample-rate'))
208 s_overlap = int(cp.get('inspiral', 'segment-overlap'))
209
210 blockDuration = ( s_length * s_num - ( s_num - 1 ) * s_overlap ) / s_rate
211 else:
212 raise ValueError, "EXIT: Cannot find block-duration in [data] section of configuration file! Either set block-duration or use --extend option."
213
214
215 minscilength = blockDuration + 2 * pad_data
216 quanta = segmentDuration / 2
217
218
219 print ">> Minimum science segment length is: %ss"%minscilength
220 print
221 if extend:
222 print ">> Will extend minimum science segment by quanta of: %ss"%quanta
223 print
224
225
226
227
228
229 if not useold:
230
231 query_start = int(grb_time - offset)
232 query_end = int(grb_time + offset)
233 for ifo in ifolist:
234 if cp.has_option('segments','%s-segments'%ifo.lower()):
235 segmentName = cp.get('segments','%s-segments'%ifo.lower())
236 check_segment_availability(grb_name, grb_time, query_start, query_end, offset, ifo, segmentName)
237
238
239
240
241
242 if not useold:
243
244 veto_file_url = cp.get('exttrig','cvs_veto_definer')
245 veto_file_path,headers = urllib.urlretrieve(veto_file_url,os.path.basename(veto_file_url))
246
247
248 deltat = 500
249 args = {'start_time' : int(grb_time - offset - deltat),
250 'end_time' : int(grb_time + offset + deltat),
251 'veto_file_path' : veto_file_path}
252 cmd = "ligolw_segments_from_cats --database --veto-file={veto_file_path} --separate-categories --gps-start-time {start_time} --gps-end-time {end_time} --output-dir=. --individual-results".format(**args)
253 print '>>',cmd
254 print
255 process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
256 output,err = process.communicate()
257
258
259 veto_files = glob.glob('./*VETOTIME_CAT*{start_time}*xml'.format(**args))
260 for filename in veto_files:
261 p = filename.split('-')
262 newname = "%s-%s_grb%s.xml"%(p[0], p[1], grb_name)
263 shutil.move(filename, newname)
264
265
266
267
268
269
270 onsource = [grb_time - int(cp.get('exttrig','onsource_left')),\
271 grb_time + int(cp.get('exttrig','onsource_right'))]
272 onSourceSegment = segments.segment(onsource[0], onsource[1])
273
274
275
276 basic_segdict = segdict = segments.segmentlistdict()
277 for ifo in ifolist:
278
279 if not cp.has_option('segments','%s-segments' % ifo.lower()):
280 continue
281
282 ifo_segfile = '%s-science_grb%s.txt' %(ifo, grb_name)
283 if os.path.exists(ifo_segfile):
284 tmplist = segmentsUtils.fromsegwizard(open(ifo_segfile))
285 try:
286 s = tmplist.find(onSourceSegment)
287 except ValueError:
288
289 continue
290 if abs(tmplist[s]) >= minscilength:
291 segdict[ifo] = segments.segmentlist([tmplist[s]])
292 basic_segdict[ifo] = segments.segmentlist([s for s in tmplist])
293 ifolist = segdict.keys()
294
295 if len(ifolist) < 2:
296 print "EXIT: Less than 2 interferometers have available data!"
297 sys.exit()
298
299
300
301
302
303
304 print ">> Vetoes that overlap with science segments:"
305 for ifo in ifolist:
306
307 cat_flag = True
308 for cat in catlist:
309
310 xmlsegfile = "./%s-VETOTIME_CAT%s_grb%s.xml" %(ifo, cat, grb_name)
311 if os.path.exists(xmlsegfile) and cat_flag:
312 testseg = segments.segment([segdict[ifo][0][0],segdict[ifo][0][1]])
313 list_overlaps = []
314
315
316 xmldoc = utils.load_filename(xmlsegfile, gz = False, contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler))
317 segs = lsctables.SegmentTable.get_table(xmldoc)
318 segdefs = lsctables.SegmentDefTable.get_table(xmldoc)
319
320
321 defdict = {}
322 for segdef in segdefs:
323 defdict[segdef.segment_def_id] = segdef.name
324
325
326 for seg in segs:
327 s = segments.segment(seg.start_time, seg.end_time)
328 if testseg.intersects(s):
329 id = seg.segment_def_id
330 list_overlaps.append([defdict[id], seg.start_time, seg.end_time])
331
332
333 for name, segstart, segend in list_overlaps:
334 print "CAT%s IFO %s, Start: %d End: %d because %s"%(cat, ifo, segstart, segend, name)
335 s = segments.segment(segstart, segend)
336 if onSourceSegment.intersects(s):
337 segdict.pop(ifo, None)
338 cat_flag = False
339 break
340 if cat == 1:
341 vetoes = segments.segmentlist(segments.segment(s[1], s[2]) for s in list_overlaps)
342 segdict[ifo] -= vetoes
343
344
345 ifolist = segdict.keys()
346
347 print
348
349 if len(ifolist) < 2:
350 print "EXIT: After vetoes, less than 2 interferometers have available data!"
351 sys.exit()
352
353
354
355
356
357
358 def sensitivity_cmp(ifo1, ifo2):
359 return cmp(sensitivity_dict[ifo1], sensitivity_dict[ifo2])
360 ifolist.sort(sensitivity_cmp)
361
362
363
364
365
366 test_combos = itertools.chain(*itertools.imap(lambda n: iterutils.choices(ifolist, n), xrange(len(ifolist), 1, -1)))
367 off_source_segment = None
368 the_ifo_combo = []
369 for ifo_combo in test_combos:
370
371 trial_seglist = segdict.intersection(ifo_combo)
372 if abs(trial_seglist) < minscilength:
373 print "EXIT: IFOs do not overlap enough for minscilength",abs(trial_seglist)
374 sys.exit()
375 else:
376 pass
377
378
379 try:
380 super_seg = trial_seglist[trial_seglist.find(onSourceSegment)].contract(pad_data)
381 except ValueError:
382 print "EXIT: ValueError with super_seg"
383 sys.exit()
384 if onSourceSegment not in super_seg:
385 print "EXIT: onsource not in super_seg"
386 sys.exit()
387
388
389 tplus = (super_seg[1] - onSourceSegment[1])
390 tminus = (onSourceSegment[0] - super_seg[0])
391
392
393 tmin = ( minscilength - 2*pad_data - abs(onSourceSegment) )
394
395
396 if tplus + tminus > tmin:
397 half_max = tmin // 2
398 if tplus < half_max:
399 print ">> Left sticks out so cut it."
400 remainder = tmin - tplus
401 tminus = min(remainder, tminus)
402 elif tminus < half_max:
403 print ">> Right sticks out so cut it."
404 remainder = tmin - tminus
405 tplus = min(remainder, tplus)
406 else:
407 print ">> Both sides stick out so cut as symmetrically as possible."
408 tminus = half_max
409 tplus = tmin - half_max
410 if tplus + tminus < tmin:
411 offsource = None
412 temp_segment = segments.segment((onSourceSegment[0] - tminus - pad_data, onSourceSegment[1] + tplus + pad_data))
413
414 if temp_segment is not None:
415 offsource = temp_segment
416 ifolist = list(ifo_combo)
417
418 if extend:
419
420 begin_time = offsource[0] - quanta * (abs(super_seg[0]-offsource[0])//quanta)
421 end_time = offsource[1] + quanta * (abs(super_seg[1]-offsource[1])//quanta)
422 offsource = segments.segment((begin_time,end_time))
423
424 break
425 print
426
427
428 if abs(offsource) < minscilength:
429 print abs(offsource),minscilength
430 print "EXIT: Calculated offsource segment but less than minscilength!"
431 sys.exit()
432
433
434 if len(ifolist) < 2:
435 print "EXIT: Calculated offsource segment but less than two IFOs!"
436 sys.exit()
437
438
439 if abs(offsource[0]-onsource[0]) < pad_data or abs(offsource[1]-onsource[1]) < pad_data:
440 print "WARNING: GRB time close to edge of offsource. Its within the padding time."
441
442
443 ifolist.sort()
444 ifotag = "".join(ifolist)
445 print ">> Offsource segment for %s GRB is:"%ifotag
446 print "Start:",offsource[0],"End:",offsource[1],"Duration:",offsource[1]-offsource[0],"Left:",grb_time-offsource[0],"Right:",offsource[1]-grb_time
447 print
448
449
450
451
452
453
454
455 for ifo in basic_ifolist:
456 if ifo in ifolist:
457 analysisFP = open('%s-analyse_grb%s.txt' %(ifo,grb_name),'w')
458 analysisFP.write('# seg\t start \t stop \t duration\n')
459 analysisFP.write('0\t %d\t %d\t %d\n' %(offsource[0],offsource[1],offsource[1]-offsource[0]))
460 else:
461 analysisFP = open('%s-analyse_grb%s.txt' %(ifo,grb_name),'w')
462 analysisFP.write('# seg\t start \t stop \t duration\n')
463
464
465 blockDuration = int(abs(offsource[0]-offsource[1])) - 2 * pad_data
466
467
468
469
470 min_psdDuration = int(cp.get('exttrig', 'min-psd-length'))
471 psdRatio = int(cp.get('exttrig', 'psd-ratio'))
472 psdDuration = 2**int(numpy.log2(blockDuration/psdRatio))
473 if psdDuration < min_psdDuration:
474 print "EXIT: PSD segment duration is too short. It is %ds but needs to be at least %ds in length."%(psdDuration,min_psdDuration)
475 sys.exit()
476
477
478 if cp.has_option('data', 'segment-duration'):
479 cp.remove_option('data', 'segment-duration')
480 cp.remove_option('data', 'block-duration')
481
482
483 print ">> Using sample rate of %d to calculate inputs for downstream processes."%sampleRate
484 print
485 segmentLength = segmentDuration*sampleRate
486 segmentCount = blockDuration/(segmentDuration/2) - 1
487 segmentOverlap = segmentLength/2
488 cp.set('data', 'segment-length', segmentLength)
489 cp.set('data', 'number-of-segments', segmentCount)
490 cp.set('inspiral', 'segment-overlap', segmentOverlap)
491
492
493 cp.set('coh_PTF_inspiral', 'block-duration', blockDuration)
494 cp.set('coh_PTF_inspiral', 'segment-duration', segmentDuration)
495 cp.set('coh_PTF_inspiral', 'psd-segment-duration', psdDuration)
496 cp.set('coh_PTF_inspiral', 'pad-data', pad_data)
497 f = open('grb%s.ini'%grb_name,'w')
498 cp.write(f)
499 f.close()
500 print ">> The [data] section of the configuration file has been edited with the following values:"
501 print "sample-rate=",sampleRate
502 print "segment-length=",segmentLength
503 print "number-of-segments=",segmentCount
504 print "segment-overlap=",segmentOverlap
505 print
506 print ">> The [coh_PTF_inspiral] section of the configuration file has been edited with the following values:"
507 print "block-duration =",blockDuration
508 print "segment-duration =",segmentDuration
509 print "psd-segment-duration =",psdDuration
510 print "pad-data =",pad_data
511 print
512
513
514 offSourceSegment = segments.segment(offsource[0], offsource[1])
515 plot_window = segments.segment(grb_time-offset, grb_time+offset)
516 plot_segments(basic_segdict, onSourceSegment, offSourceSegment, grb_time, plot_window, "segment_plot_%s.png"%grb_name, grb_name)
517
518
519 if make_xml:
520
521 xmldoc = ligolw.Document()
522 xmldoc.appendChild(ligolw.LIGO_LW())
523 tbl = lsctables.New(lsctables.ExtTriggersTable)
524 xmldoc.childNodes[-1].appendChild(tbl)
525
526
527 row = lsctables.ExtTriggersTable()
528 row.process_id = None
529 row.det_alts = None
530 row.det_band = None
531 row.det_fluence = None
532 row.det_fluence_int = None
533 row.det_name = None
534 row.det_peak = None
535 row.det_peak_int = None
536 row.det_snr = ''
537 row.email_time = 0
538 row.event_dec = float(grb_dec)
539 row.event_dec_err = 0.0
540 row.event_epoch = ''
541 row.event_err_type = ''
542 row.event_ra = float(grb_ra)
543 row.event_ra_err = 0.0
544 row.start_time = grb_time
545 row.start_time_ns = 0
546 row.event_type = ''
547 row.event_z = 0.0
548 row.event_z_err = 0.0
549 row.notice_comments = ''
550 row.notice_id = ''
551 row.notice_sequence = ''
552 row.notice_time = 0
553 row.notice_type = ''
554 row.notice_url = ''
555 row.obs_fov_dec = 0.0
556 row.obs_fov_dec_width = 0.0
557 row.obs_fov_ra = 0.0
558 row.obs_fov_ra_width = 0.0
559 row.obs_loc_ele = 0.0
560 row.obs_loc_lat = 0.0
561 row.obs_loc_long = 0.0
562 row.ligo_fave_lho = 0.0
563 row.ligo_fave_llo = 0.0
564 row.ligo_delay = 0.0
565 row.event_number_gcn = 9999
566 row.event_number_grb = grb_name
567 row.event_status = 0
568
569
570 tbl.extend([row])
571 filename = 'grb%s.xml' % grb_name
572 utils.write_filename(xmldoc, filename)
573
574
575 if make_plots:
576 vetodict = segments.segmentlistdict()
577 for cat in catlist:
578 for ifo in ifolist:
579 vetofile = "%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name)
580 xmldoc = utils.load_filename(vetofile, gz = False, contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler))
581 segs = lsctables.SegmentTable.get_table(xmldoc)
582 segdefs = lsctables.SegmentDefTable.get_table(xmldoc)
583 vetodict[ifo] = segments.segmentlist(segments.segment(s.start_time, s.end_time) for s in segs)
584
585 if vetodict:
586 plot_segments(vetodict, onSourceSegment, offSourceSegment, grb_time, plot_window, "veto_plot_CAT%s_%s.png"%(cat,grb_name), "%s CAT%s"%(grb_name, cat))
587
588
589 return 'grb%s.ini'%grb_name, ifolist, onSourceSegment, offSourceSegment
590
591 if __name__ == "__main__":
592
593 opts,args = parse_args()
594
595
596 if opts.grb_file:
597 xmldoc = utils.load_filename(opts.grb_file, gz=opts.grb_file.endswith('.gz'), contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler))
598 ext_table = lsctables.ExtTriggersTable.get_table(xmldoc)
599 grb_time = ext_table[0].start_time
600 grb_name = os.path.basename(opts.grb_file)[3:-4]
601 grb_ra = ext_table[0].event_ra
602 grb_dec = ext_table[0].event_dec
603 else:
604 grb_name = opts.name
605 grb_time = int(opts.time)
606 grb_ra = float(opts.ra)
607 grb_dec = float(opts.dec)
608
609
610 exttrig_dataquery(grb_name, grb_time, grb_ra, grb_dec, opts.offset, opts.config_file, opts.extend, opts.useold, opts.make_plots, opts.make_xml)
611