Package glue :: Module LDBDWClient
[hide private]
[frames] | no frames]

Source Code for Module glue.LDBDWClient

  1  """ 
  2  The LDBDWClient module provides an API for connecting to and making requests of a LDBDWServer. 
  3   
  4  This file is part of the Grid LSC User Environment (GLUE) 
  5   
  6  GLUE is free software: you can redistribute it and/or modify it under the 
  7  terms of the GNU General Public License as published by the Free Software 
  8  Foundation, either version 3 of the License, or (at your option) any later 
  9  version. 
 10   
 11  This program is distributed in the hope that it will be useful, but WITHOUT 
 12  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 13  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
 14  details. 
 15   
 16  You should have received a copy of the GNU General Public License along with 
 17  this program.  If not, see <http://www.gnu.org/licenses/>. 
 18  """ 
 19   
 20  from glue import git_version 
 21  __date__ = git_version.date 
 22  __version__ = git_version.id 
 23   
 24  import sys 
 25  import os 
 26  import types 
 27  import re 
 28  import six.moves.cPickle 
 29  import xml.parsers.expat 
 30  import six.moves.http_client 
 31   
 32  try: 
 33      from cjson import (decode, encode) 
 34  except ImportError: 
 35      from json import (loads as decode, dumps as encode) 
 36   
 37  try: 
 38      import M2Crypto 
 39  except ImportError as e: 
 40      sys.stderr.write(""" 
 41  ligo_data_find requires M2Crypto 
 42   
 43  On CentOS 5 and other RHEL based platforms 
 44  this package is available from the EPEL 
 45  repository by doing 
 46   
 47  yum install m2crypto 
 48   
 49  For Debian Lenny this package is available 
 50  by doing 
 51   
 52  apt-get install python-m2crypto 
 53   
 54  Mac OS X users can find this package in 
 55  MacPorts. 
 56   
 57  %s 
 58  """ % e) 
 59      sys.exit(1) 
 60   
 61   
62 -def version():
63 return __version__
64 65
66 -class SimpleLWXMLParser:
67 """ 68 A very simple LIGO_LW XML parser class that reads the only keeps 69 tables that do not contain the strings sngl_ or multi_ 70 71 The class is not very robust as can have problems if the line 72 breaks do not appear in the standard places in the XML file. 73 """
74 - def __init__(self):
75 """ 76 Constructs an instance. 77 78 The private variable ignore_pat determines what tables we ignore. 79 """ 80 self.__p = xml.parsers.expat.ParserCreate() 81 self.__in_table = 0 82 self.__silent = 0 83 self.__ignore_pat = re.compile(r'.*(sngl_|multi_).*', re.IGNORECASE) 84 self.__p.StartElementHandler = self.start_element 85 self.__p.EndElementHandler = self.end_element
86
87 - def __del__(self):
88 """ 89 Destroys an instance by shutting down and deleting the parser. 90 """ 91 self.__p("",1) 92 del self.__p
93
94 - def start_element(self, name, attrs):
95 """ 96 Callback for start of an XML element. Checks to see if we are 97 about to start a table that matches the ignore pattern. 98 99 @param name: the name of the tag being opened 100 @type name: string 101 102 @param attrs: a dictionary of the attributes for the tag being opened 103 @type attrs: dictionary 104 """ 105 if name.lower() == "table": 106 for attr in attrs.keys(): 107 if attr.lower() == "name": 108 if self.__ignore_pat.search(attrs[attr]): 109 self.__in_table = 1
110
111 - def end_element(self, name):
112 """ 113 Callback for the end of an XML element. If the ignore flag is 114 set, reset it so we start outputing the table again. 115 116 @param name: the name of the tag being closed 117 @type name: string 118 """ 119 if name.lower() == "table": 120 if self.__in_table: 121 self.__in_table = 0
122
123 - def parse_line(self, line):
124 """ 125 For each line we are passed, call the XML parser. Returns the 126 line if we are outside one of the ignored tables, otherwise 127 returns the empty string. 128 129 @param line: the line of the LIGO_LW XML file to be parsed 130 @type line: string 131 132 @return: the line of XML passed in or the null string 133 @rtype: string 134 """ 135 self.__p.Parse(line) 136 if self.__in_table: 137 self.__silent = 1 138 if not self.__silent: 139 ret = line 140 else: 141 ret = "" 142 if not self.__in_table: 143 self.__silent = 0 144 return ret
145
146 -def findCredential():
147 """ 148 Follow the usual path that GSI libraries would 149 follow to find a valid proxy credential but 150 also allow an end entity certificate to be used 151 along with an unencrypted private key if they 152 are pointed to by X509_USER_CERT and X509_USER_KEY 153 since we expect this will be the output from 154 the eventual ligo-login wrapper around 155 kinit and then myproxy-login. 156 """ 157 158 # use X509_USER_PROXY from environment if set 159 if 'X509_USER_PROXY' in os.environ: 160 filePath = os.environ['X509_USER_PROXY'] 161 if validateProxy(filePath): 162 return filePath, filePath 163 else: 164 RFCproxyUsage() 165 sys.exit(1) 166 167 # use X509_USER_CERT and X509_USER_KEY if set 168 if 'X509_USER_CERT' in os.environ: 169 if 'X509_USER_KEY' in os.environ: 170 certFile = os.environ['X509_USER_CERT'] 171 keyFile = os.environ['X509_USER_KEY'] 172 return certFile, keyFile 173 174 # search for proxy file on disk 175 uid = os.getuid() 176 path = "/tmp/x509up_u%d" % uid 177 178 if os.access(path, os.R_OK): 179 if validateProxy(path): 180 return path, path 181 else: 182 RFCproxyUsage() 183 sys.exit(1) 184 185 # if we get here could not find a credential 186 RFCproxyUsage() 187 sys.exit(1)
188
189 -def validateProxy(path):
190 """ 191 Test that the proxy certificate is RFC 3820 192 compliant and that it is valid for at least 193 the next 15 minutes. 194 """ 195 196 # load the proxy from path 197 try: 198 proxy = M2Crypto.X509.load_cert(path) 199 except Exception as e: 200 msg = "Unable to load proxy from path %s : %s\n" % (path, e) 201 sys.stderr.write(msg) 202 sys.exit(1) 203 204 # make sure the proxy is RFC 3820 compliant 205 # or is an end-entity X.509 certificate 206 try: 207 proxy.get_ext("proxyCertInfo") 208 except LookupError: 209 # it is not an RFC 3820 proxy so check 210 # if it is an old globus legacy proxy 211 subject = proxy.get_subject().as_text() 212 if re.search(r'.+CN=proxy$', subject): 213 # it is so print warning and exit 214 RFCproxyUsage() 215 sys.exit(1) 216 217 # attempt to make sure the proxy is still good for more than 15 minutes 218 try: 219 expireASN1 = proxy.get_not_after().__str__() 220 expireGMT = time.strptime(expireASN1, "%b %d %H:%M:%S %Y %Z") 221 expireUTC = calendar.timegm(expireGMT) 222 now = int(time.time()) 223 secondsLeft = expireUTC - now 224 except Exception as e: 225 # problem getting or parsing time so just let the client 226 # continue and pass the issue along to the server 227 secondsLeft = 3600 228 229 if secondsLeft <= 0: 230 msg = """\ 231 Your proxy certificate is expired. 232 233 Please generate a new proxy certificate and 234 try again. 235 """ 236 sys.stderr.write(msg) 237 sys.exit(1) 238 239 if secondsLeft < (60 * 15): 240 msg = """\ 241 Your proxy certificate expires in less than 242 15 minutes. 243 244 Please generate a new proxy certificate and 245 try again. 246 """ 247 sys.stderr.write(msg) 248 sys.exit(1) 249 250 # return True to indicate validated proxy 251 return True
252 253
254 -def RFCproxyUsage():
255 """ 256 Print a simple error message about not finding 257 a RFC 3820 compliant proxy certificate. 258 """ 259 msg = """\ 260 Could not find a valid proxy credential. 261 LIGO users, please run 'ligo-proxy-init' and try again. 262 Others, please run 'grid-proxy-init' and try again. 263 """ 264 265 sys.stderr.write(msg)
266 267
268 -class LDBDClientException(Exception):
269 """Exceptions returned by server""" 270 pass
271 272
273 -class LDBDClient(object):
274 - def __init__(self, host, port = None, protocol = None, identity = None):
275 """ 276 """ 277 self.host = host 278 self.port = port 279 self.protocol = protocol 280 281 if self.port: 282 self.server = "%s:%d" % (self.host, self.port) 283 else: 284 self.server = self.host 285 286 # this is a holdover from the pyGlobus era and is not 287 # acutally used at this time since we do not check the 288 # credential of the server 289 self.identity = identity 290 291 # find credential unless the protocol is http: 292 if protocol == "http": 293 self.certFile = None 294 self.keyFile = None 295 else: 296 self.certFile, self.keyFile = findCredential()
297
298 - def ping(self):
299 """ 300 """ 301 server = self.server 302 protocol = self.protocol 303 304 if protocol == "https": 305 #if self.certFile and self.keyFile: 306 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile) 307 else: 308 h = six.moves.http_client.HTTPConnection(server) 309 310 url = "/ldbd/ping.json" 311 headers = {"Content-type" : "application/json"} 312 data = "" 313 body = encode(protocol) 314 315 try: 316 h.request("POST", url, body, headers) 317 response = h.getresponse() 318 except Exception as e: 319 msg = "Error pinging server %s: %s" % (server, e) 320 raise LDBDClientException(msg) 321 322 if response.status != 200: 323 msg = "Server returned code %d: %s" % (response.status, response.reason) 324 body = response.read() 325 msg += body 326 raise LDBDClientException(msg) 327 328 # since status is 200 OK the ping was good 329 body = response.read() 330 msg = decode(body) 331 332 return msg
333
334 - def query(self,sql):
335 """ 336 """ 337 338 server = self.server 339 protocol = self.protocol 340 if protocol == "https": 341 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile) 342 else: 343 h = six.moves.http_client.HTTPConnection(server) 344 345 url = "/ldbd/query.json" 346 headers = {"Content-type" : "application/json"} 347 body = encode(protocol + ":" + sql) 348 349 try: 350 h.request("POST", url, body, headers) 351 response = h.getresponse() 352 except Exception as e: 353 msg = "Error querying server %s: %s" % (server, e) 354 raise LDBDClientException(msg) 355 356 if response.status != 200: 357 msg = "Server returned code %d: %s" % (response.status, response.reason) 358 body = response.read() 359 msg += body 360 raise LDBDClientException(msg) 361 362 # since status is 200 OK the query was good 363 body = response.read() 364 msg = decode(body) 365 366 return msg
367
368 - def insert(self,xmltext):
369 """ 370 """ 371 372 server = self.server 373 protocol = self.protocol 374 if protocol == "https": 375 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile) 376 else: 377 msg = "Insecure connection DOES NOT surpport INSERT." 378 msg += '\nTo INSERT, authorized users please specify protocol "https" in your --segment-url argument.' 379 msg += '\nFor example, "--segment-url https://segdb.ligo.caltech.edu".' 380 raise LDBDClientException(msg) 381 382 url = "/ldbd/insert.json" 383 headers = {"Content-type" : "application/json"} 384 body = encode(xmltext) 385 386 try: 387 h.request("POST", url, body, headers) 388 response = h.getresponse() 389 except Exception as e: 390 msg = "Error querying server %s: %s" % (server, e) 391 raise LDBDClientException(msg) 392 393 if response.status != 200: 394 msg = "Server returned code %d: %s" % (response.status, response.reason) 395 body = response.read() 396 msg += body 397 raise LDBDClientException(msg) 398 399 # since status is 200 OK the query was good 400 body = response.read() 401 msg = decode(body) 402 403 return msg
404
405 - def insertmap(self,xmltext,lfnpfn_dict):
406 """ 407 """ 408 server = self.server 409 protocol = self.protocol 410 if protocol == "https": 411 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile) 412 else: 413 msg = "Insecure connection DOES NOT surpport INSERTMAP." 414 msg += '\nTo INSERTMAP, authorized users please specify protocol "https" in your --segment-url argument.' 415 msg += '\nFor example, "--segment-url https://segdb.ligo.caltech.edu".' 416 raise LDBDClientException(msg) 417 418 url = "/ldbd/insertmap.json" 419 headers = {"Content-type" : "application/json"} 420 421 pmsg = six.moves.cPickle.dumps(lfnpfn_dict) 422 data = [xmltext, pmsg] 423 body = encode(data) 424 425 try: 426 h.request("POST", url, body, headers) 427 response = h.getresponse() 428 except Exception as e: 429 msg = "Error querying server %s: %s" % (server, e) 430 raise LDBDClientException(msg) 431 432 if response.status != 200: 433 msg = "Server returned code %d: %s" % (response.status, response.reason) 434 body = response.read() 435 msg += body 436 raise LDBDClientException(msg) 437 438 # since status is 200 OK the query was good 439 body = response.read() 440 msg = decode(body) 441 442 return msg
443 444
445 - def insertdmt(self,xmltext):
446 """ 447 """ 448 server = self.server 449 protocol = self.protocol 450 if protocol == "https": 451 h = six.moves.http_client.HTTPSConnection(server, key_file = self.keyFile, cert_file = self.certFile) 452 else: 453 msg = "Insecure connection DOES NOT surpport INSERTDMT." 454 msg += '\nTo INSERTDMT, authorized users please specify protocol "https" in your --segment-url argument.' 455 msg += '\nFor example, "--segment-url https://segdb.ligo.caltech.edu".' 456 raise LDBDClientException(msg) 457 458 url = "/ldbd/insertdmt.json" 459 headers = {"Content-type" : "application/json"} 460 body = encode(xmltext) 461 462 try: 463 h.request("POST", url, body, headers) 464 response = h.getresponse() 465 except Exception as e: 466 msg = "Error querying server %s: %s" % (server, e) 467 raise LDBDClientException(msg) 468 469 if response.status != 200: 470 msg = "Server returned code %d: %s" % (response.status, response.reason) 471 body = response.read() 472 msg += body 473 raise LDBDClientException(msg) 474 475 # since status is 200 OK the query was good 476 body = response.read() 477 msg = decode(body) 478 479 return msg
480