CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
BeamSpotWorkflow Namespace Reference

Functions

def aselectFilesToProcess
 
def compareLumiLists
 
def getLastClosedRun
 
def getLastUploadedIOV
 General functions. More...
 
def getListOfFilesToProcess
 
def getListOfRunsAndLumiFromDBS
 
def getListOfRunsAndLumiFromFile
 
def getListOfRunsAndLumiFromRR
 
def getNewRunList
 
def getNumberOfFilesToProcessForRun
 
def getRunNumberFromDBSName
 
def getRunNumberFromFileName
 
def main
 
def removeUncompleteRuns
 
def selectFilesToProcess
 

Variables

string error = "Please set a crab environment in order to get the proper JSON lib"
 

Function Documentation

def BeamSpotWorkflow.aselectFilesToProcess (   listOfFilesToProcess,
  newRunList 
)

Definition at line 491 of file BeamSpotWorkflow.py.

References beamvalidation.exit(), getLastClosedRun(), getRunNumberFromDBSName(), getRunNumberFromFileName(), print(), and str.

491 
492 def aselectFilesToProcess(listOfFilesToProcess,newRunList):
493  selectedFiles = []
494  runsToProcess = {}
495  processedRuns = {}
496  for file in listOfFilesToProcess:
497  run = getRunNumberFromDBSName(file)
498 # print "To process: " + str(run)
499  if run not in runsToProcess:
500  runsToProcess[run] = 1
501  else:
502  runsToProcess[run] = runsToProcess[run] + 1
503 
504  for file in newRunList:
505  run = getRunNumberFromFileName(file)
506 # print "Processed: " + str(run)
507  if run not in processedRuns:
508  processedRuns[run] = 1
509  else:
510  processedRuns[run] = processedRuns[run] + 1
511 
512  #WARNING: getLastClosedRun MUST also have a timeout otherwise the last run will not be considered
513  lastClosedRun = getLastClosedRun(listOfFilesToProcess)
514 # print "LastClosedRun:-" + str(lastClosedRun) + "-"
515 
516  processedRunsKeys = sorted(processedRuns.keys())
517 
518  for run in processedRunsKeys:
519  if run <= lastClosedRun :
520  print("For run " + str(run) + " I have processed " + str(processedRuns[run]) + " files and in DBS there are " + str(runsToProcess[run]) + " files!")
521  if not run in runsToProcess:
522  exit("ERROR: I have a result file for run " + str(run) + " but it doesn't exist in DBS. Impossible but it happened!")
523  lumiList = getDBSLumiListForRun(run)
524  if processedRuns[run] == runsToProcess[run]:
525  for file in newRunList:
526  if run == getRunNumberFromFileName(file):
527  selectedFiles.append(file)
528  else:
529  exit("ERROR: For run " + str(run) + " I have processed " + str(processedRuns[run]) + " files but in DBS there are " + str(runsToProcess[run]) + " files!")
530  return selectedFiles
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.compareLumiLists (   listA,
  listB,
  errors = [],
  tolerance = 0 
)

Definition at line 436 of file BeamSpotWorkflow.py.

References str.

Referenced by selectFilesToProcess().

436 
437 def compareLumiLists(listA,listB,errors=[],tolerance=0):
438  lenA = len(listA)
439  lenB = len(listB)
440  if lenA < lenB-(lenB*float(tolerance)/100):
441  errors.append("ERROR: The number of lumi sections is different: listA(" + str(lenA) + ")!=(" + str(lenB) + ")listB")
442  #else:
443  #errors.append("Lumi check ok!listA(" + str(lenA) + ")-(" + str(lenB) + ")listB")
444  #print errors
445  listA.sort()
446  listB.sort()
447  #shorter = lenA
448  #if lenB < shorter:
449  # shorter = lenB
450  #a = 0
451  #b = 0
452  badA = []
453  badB = []
454  #print listB
455  #print listA
456  #print len(listA)
457  #print len(listB)
458  #counter = 1
459  for lumi in listA:
460  #print str(counter) + "->" + str(lumi)
461  #counter += 1
462  if not lumi in listB:
463  errors.append("Lumi (" + str(lumi) + ") is in listA but not in listB")
464  badB.append(lumi)
465  #print "Bad B: " + str(lumi)
466  #exit("hola")
467  for lumi in listB:
468  if not lumi in listA:
469  errors.append("Lumi (" + str(lumi) + ") is in listB but not in listA")
470  badA.append(lumi)
471  #print "Bad A: " + str(lumi)
472 
473  return badA,badB
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.getLastClosedRun (   DBSListOfFiles)

Definition at line 219 of file BeamSpotWorkflow.py.

References getRunNumberFromDBSName().

Referenced by aselectFilesToProcess().

219 
220 def getLastClosedRun(DBSListOfFiles):
221  runs = []
222  for file in DBSListOfFiles:
223  runNumber = getRunNumberFromDBSName(file)
224  if runs.count(runNumber) == 0:
225  runs.append(runNumber)
226 
227  if len(runs) <= 1: #No closed run
228  return -1
229  else:
230  runs.sort()
231  return long(runs[len(runs)-2])
def BeamSpotWorkflow.getLastUploadedIOV (   tagName,
  destDB = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT" 
)

General functions.

Definition at line 58 of file BeamSpotWorkflow.py.

References beamvalidation.exit(), spr.find(), and print().

Referenced by main().

58 
59 def getLastUploadedIOV(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
60  #return 582088327592295
61  listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName
62  dbError = subprocess.getstatusoutput( listIOVCommand )
63  if dbError[0] != 0 :
64  if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
65  print("Creating a new tag because I got the following error contacting the DB")
66  print(dbError[1])
67  return 1
68  #return 133928
69  else:
70  exit("ERROR: Can\'t connect to db because:\n" + dbError[1])
71 
72 
73  aCommand = listIOVCommand + " | grep DB= | tail -1 | awk \'{print $1}\'"
74  output = subprocess.getstatusoutput( aCommand )
75 
76  #WARNING when we pass to lumi IOV this should be long long
77  if output[1] == '':
78  exit("ERROR: The tag " + tagName + " exists but I can't get the value of the last IOV")
79 
80  return long(output[1])
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
BeamSpotWorkflow.getLastUploadedIOV
def getLastUploadedIOV
General functions.
Definition: BeamSpotWorkflow.py:58
def BeamSpotWorkflow.getListOfFilesToProcess (   dataSet,
  lastRun = -1 
)

Definition at line 82 of file BeamSpotWorkflow.py.

References submitPVValidationJobs.split(), and str.

82 
83 def getListOfFilesToProcess(dataSet,lastRun=-1):
84  queryCommand = "dbs --search --query \"find file where dataset=" + dataSet
85  if lastRun != -1:
86  queryCommand = queryCommand + " and run > " + str(lastRun)
87  queryCommand = queryCommand + "\" | grep .root"
88 # print " >> " + queryCommand
89  output = subprocess.getstatusoutput( queryCommand )
90  return output[1].split('\n')
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.getListOfRunsAndLumiFromDBS (   dataSet,
  lastRun = -1 
)

Definition at line 102 of file BeamSpotWorkflow.py.

References bitset_utilities.append(), beamvalidation.exit(), spr.find(), print(), sistrip::SpyUtilities.range(), submitPVValidationJobs.split(), and str.

Referenced by main().

102 
103 def getListOfRunsAndLumiFromDBS(dataSet,lastRun=-1):
104  datasetList = dataSet.split(',')
105  outputList = []
106  for data in datasetList:
107  queryCommand = "dbs --search --query \"find run,lumi where dataset=" + data
108  if lastRun != -1:
109  queryCommand = queryCommand + " and run > " + str(lastRun)
110  queryCommand += "\""
111  print(" >> " + queryCommand)
112  output = []
113  for i in range(0,3):
114  output = subprocess.getstatusoutput( queryCommand )
115  if output[0] == 0 and not (output[1].find("ERROR") != -1 or output[1].find("Error") != -1) :
116  break
117  if output[0] != 0:
118  exit("ERROR: I can't contact DBS for the following reason:\n" + output[1])
119  #print output[1]
120  tmpList = output[1].split('\n')
121  for file in tmpList:
122  outputList.append(file)
123  runsAndLumis = {}
124  for out in outputList:
125  regExp = re.search('(\d+)\s+(\d+)',out)
126  if regExp:
127  run = long(regExp.group(1))
128  lumi = long(regExp.group(2))
129  if not run in runsAndLumis:
130  runsAndLumis[run] = []
131  runsAndLumis[run].append(lumi)
132 
133 # print runsAndLumis
134 # exit("ok")
135  return runsAndLumis
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.getListOfRunsAndLumiFromFile (   firstRun = -1,
  fileName = "" 
)

Definition at line 137 of file BeamSpotWorkflow.py.

Referenced by main().

137 
138 def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
139  file = open(fileName);
140  jsonFile = file.read();
141  file.close()
142  jsonList=json.loads(jsonFile);
143 
144  selected_dcs = {};
145  for element in jsonList:
146  selected_dcs[long(element)]=jsonList[element]
147  return selected_dcs
def BeamSpotWorkflow.getListOfRunsAndLumiFromRR (   firstRun = -1)

Definition at line 149 of file BeamSpotWorkflow.py.

References print(), and str.

Referenced by main().

149 
150 def getListOfRunsAndLumiFromRR(firstRun=-1):
151  RunReg ="http://pccmsdqm04.cern.ch/runregistry"
152  #RunReg = "http://localhost:40010/runregistry"
153  #Dataset=%Online%
154  Group = "Collisions10"
155 
156  # get handler to RR XML-RPC server
157  FULLADDRESS=RunReg + "/xmlrpc"
158  #print "RunRegistry from: ",FULLADDRESS
159  server = xmlrpclib.ServerProxy(FULLADDRESS)
160  #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
161  sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun)
162  #sel_dcstable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
163 
164  maxAttempts = 3;
165  tries = 0;
166  while tries<maxAttempts:
167  try:
168  run_data = server.DataExporter.export('RUN' , 'GLOBAL', 'csv_runs', sel_runtable)
169  #dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json' , sel_dcstable)
170  break
171  except:
172  print("Something wrong in accessing runregistry, retrying in 2s....", tries, "/", maxAttempts)
173  tries += 1
174  time.sleep(2)
175  if tries==maxAttempts:
176  error = "Run registry unaccessible.....exiting now"
177  return {};
178 
179 
180  listOfRuns=[]
181  for line in run_data.split("\n"):
182  run=line.split(',')[0]
183  if run.isdigit():
184  listOfRuns.append(run)
185 
186 
187  firstRun = listOfRuns[len(listOfRuns)-1];
188  lastRun = listOfRuns[0];
189  sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
190 
191  tries = 0;
192  while tries<maxAttempts:
193  try:
194  #run_data = server.DataExporter.export('RUN' , 'GLOBAL', 'csv_runs', sel_runtable)
195  dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json' , sel_dcstable)
196  break
197  except:
198  print("I was able to get the list of runs and now I am trying to access the detector status, retrying in 2s....", tries, "/", maxAttempts)
199  tries += 1
200  time.sleep(2)
201  if tries==maxAttempts:
202  error = "Run registry unaccessible.....exiting now"
203  return {};
204 
205  selected_dcs={}
206  jsonList=json.loads(dcs_data)
207 
208  #for element in jsonList:
209  for element in listOfRuns:
210  #if element in listOfRuns:
211  if element in jsonList:
212  selected_dcs[long(element)]=jsonList[element]
213  else:
214  print("WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!")
215  selected_dcs[long(element)]= [[]]
216  #print selected_dcs
217  return selected_dcs
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.getNewRunList (   fromDir,
  lastUploadedIOV 
)

Definition at line 248 of file BeamSpotWorkflow.py.

References getRunNumberFromFileName(), and eostools.ls().

Referenced by main().

248 
249 def getNewRunList(fromDir,lastUploadedIOV):
250  newRunList = []
251  listOfFiles = ls(fromDir,".txt")
252  runFileMap = {}
253  for fileName in listOfFiles:
254  runNumber = getRunNumberFromFileName(fileName)
255  if runNumber > lastUploadedIOV:
256  newRunList.append(fileName)
257  return newRunList
eostools.ls
def ls
Definition: eostools.py:349
def BeamSpotWorkflow.getNumberOfFilesToProcessForRun (   dataSet,
  run 
)

Definition at line 92 of file BeamSpotWorkflow.py.

References submitPVValidationJobs.split(), and str.

Referenced by removeUncompleteRuns(), and selectFilesToProcess().

92 
93 def getNumberOfFilesToProcessForRun(dataSet,run):
94  queryCommand = "dbs --search --query \"find file where dataset=" + dataSet + " and run = " + str(run) + "\" | grep .root"
95  #print " >> " + queryCommand
96  output = subprocess.getstatusoutput( queryCommand )
97  if output[0] != 0:
98  return 0
99  else:
100  return len(output[1].split('\n'))
BeamSpotWorkflow.getNumberOfFilesToProcessForRun
def getNumberOfFilesToProcessForRun
Definition: BeamSpotWorkflow.py:92
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.getRunNumberFromDBSName (   fileName)

Definition at line 241 of file BeamSpotWorkflow.py.

Referenced by aselectFilesToProcess(), and getLastClosedRun().

241 
242 def getRunNumberFromDBSName(fileName):
243  regExp = re.search('(\D+)/(\d+)/(\d+)/(\d+)/(\D+)',fileName)
244  if not regExp:
245  return -1
246  return long(regExp.group(3)+regExp.group(4))
def BeamSpotWorkflow.getRunNumberFromFileName (   fileName)

Definition at line 233 of file BeamSpotWorkflow.py.

Referenced by aselectFilesToProcess(), getNewRunList(), and removeUncompleteRuns().

233 
234 def getRunNumberFromFileName(fileName):
235 # regExp = re.search('(\D+)_(\d+)_(\d+)_(\d+)',fileName)
236  regExp = re.search('(\D+)_(\d+)_(\d+)_',fileName)
237  if not regExp:
238  return -1
239  return long(regExp.group(3))
def BeamSpotWorkflow.main ( )

Definition at line 532 of file BeamSpotWorkflow.py.

References CommonMethods.appendSqliteFile(), CommonMethods.checkLock(), CommonMethods.createWeightedPayloads(), trklet.dirExists(), CommonMethods.dumpValues(), beamvalidation.exit(), mergeVDriftHistosByStation.file, getLastUploadedIOV(), getListOfRunsAndLumiFromDBS(), getListOfRunsAndLumiFromFile(), getListOfRunsAndLumiFromRR(), getNewRunList(), if(), CommonMethods.lock(), timeUnitHelper.pack(), dumpparser.parse(), print(), sistrip::SpyUtilities.range(), CommonMethods.readBeamSpotFile(), CommonMethods.readSqliteFile(), CommonMethods.rmLock(), selectFilesToProcess(), CommonMethods.sendEmail(), CommonMethods.setLockName(), CommonMethods.sortAndCleanBeamList(), str, timeUnitHelper.unpackLumiid(), CommonMethods.uploadSqliteFile(), and CommonMethods.writeSqliteFile().

532 
533 def main():
534  ######### COMMAND LINE OPTIONS ##############
535  option,args = parse(__doc__)
536 
537  ######### Check if there is already a megascript running ########
538  if option.lock:
539  setLockName('.' + option.lock)
540  if checkLock():
541  print("There is already a megascript runnning...exiting")
542  return
543  else:
544  lock()
545 
546 
547  destDB = 'oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT'
548  if option.Test:
549  destDB = 'oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT'
550 
551  ######### CONFIGURATION FILE ################
552  cfgFile = "BeamSpotWorkflow.cfg"
553  if option.cfg:
554  cfgFile = option.cfg
555  configurationFile = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/scripts/" + cfgFile
556  configuration = ConfigParser.ConfigParser()
557  print('Reading configuration from ', configurationFile)
558  configuration.read(configurationFile)
559 
560  sourceDir = configuration.get('Common','SOURCE_DIR')
561  archiveDir = configuration.get('Common','ARCHIVE_DIR')
562  workingDir = configuration.get('Common','WORKING_DIR')
563  databaseTag = configuration.get('Common','DBTAG')
564  dataSet = configuration.get('Common','DATASET')
565  fileIOVBase = configuration.get('Common','FILE_IOV_BASE')
566  dbIOVBase = configuration.get('Common','DB_IOV_BASE')
567  dbsTolerance = float(configuration.get('Common','DBS_TOLERANCE'))
568  dbsTolerancePercent = float(configuration.get('Common','DBS_TOLERANCE_PERCENT'))
569  rrTolerance = float(configuration.get('Common','RR_TOLERANCE'))
570  missingFilesTolerance = float(configuration.get('Common','MISSING_FILES_TOLERANCE'))
571  missingLumisTimeout = float(configuration.get('Common','MISSING_LUMIS_TIMEOUT'))
572  jsonFileName = configuration.get('Common','JSON_FILE')
573  mailList = configuration.get('Common','EMAIL')
574 
575  ######### DIRECTORIES SETUP #################
576  if sourceDir[len(sourceDir)-1] != '/':
577  sourceDir = sourceDir + '/'
578  if not dirExists(sourceDir):
579  error = "ERROR: The source directory " + sourceDir + " doesn't exist!"
580  sendEmail(mailList,error)
581  exit(error)
582 
583  if archiveDir[len(archiveDir)-1] != '/':
584  archiveDir = archiveDir + '/'
585  if not os.path.isdir(archiveDir):
586  os.mkdir(archiveDir)
587 
588  if workingDir[len(workingDir)-1] != '/':
589  workingDir = workingDir + '/'
590  if not os.path.isdir(workingDir):
591  os.mkdir(workingDir)
592  else:
593  os.system("rm -f "+ workingDir + "*")
594 
595 
596  print("Getting last IOV for tag: " + databaseTag)
597  lastUploadedIOV = 1
598  if destDB == "oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT":
599  lastUploadedIOV = getLastUploadedIOV(databaseTag)
600  else:
601  lastUploadedIOV = getLastUploadedIOV(databaseTag,destDB)
602 
603  #lastUploadedIOV = 133885
604  #lastUploadedIOV = 575216380019329
605  if dbIOVBase == "lumiid":
606  lastUploadedIOV = unpackLumiid(lastUploadedIOV)["run"]
607 
608  ######### Get list of files processed after the last IOV
609  print("Getting list of files processed after IOV " + str(lastUploadedIOV))
610  newProcessedRunList = getNewRunList(sourceDir,lastUploadedIOV)
611  if len(newProcessedRunList) == 0:
612  exit("There are no new runs after " + str(lastUploadedIOV))
613 
614  ######### Copy files to archive directory
615  print("Copying files to archive directory")
616  copiedFiles = []
617  for i in range(3):
618  copiedFiles = cp(sourceDir,archiveDir,newProcessedRunList)
619  if len(copiedFiles) == len(newProcessedRunList):
620  break;
621  if len(copiedFiles) != len(newProcessedRunList):
622  error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(newProcessedRunList))
623  sendEmail(mailList,error)
624  exit(error)
625 
626 
627  ######### Get from DBS the list of files after last IOV
628  #listOfFilesToProcess = getListOfFilesToProcess(dataSet,lastUploadedIOV)
629  print("Getting list of files from DBS")
630  listOfRunsAndLumiFromDBS = getListOfRunsAndLumiFromDBS(dataSet,lastUploadedIOV)
631  if len(listOfRunsAndLumiFromDBS) == 0:
632  exit("There are no files in DBS to process")
633  print("Getting list of files from RR")
634  listOfRunsAndLumiFromRR = getListOfRunsAndLumiFromRR(lastUploadedIOV)
635  if(not listOfRunsAndLumiFromRR):
636  print("Looks like I can't get anything from the run registry so I'll get the data from the json file " + jsonFileName)
637  listOfRunsAndLumiFromRR = getListOfRunsAndLumiFromFile(lastUploadedIOV,jsonFileName)
638  ######### Get list of files to process for DB
639  #selectedFilesToProcess = selectFilesToProcess(listOfFilesToProcess,copiedFiles)
640  #completeProcessedRuns = removeUncompleteRuns(copiedFiles,dataSet)
641  #print copiedFiles
642  #print completeProcessedRuns
643  #exit("complete")
644  print("Getting list of files to process")
645  selectedFilesToProcess = selectFilesToProcess(listOfRunsAndLumiFromDBS,listOfRunsAndLumiFromRR,copiedFiles,archiveDir,dataSet,mailList,dbsTolerance,dbsTolerancePercent,rrTolerance,missingFilesTolerance,missingLumisTimeout)
646  if len(selectedFilesToProcess) == 0:
647  exit("There are no files to process")
648 
649  #print selectedFilesToProcess
650  ######### Copy files to working directory
651  print("Copying files from archive to working directory")
652  copiedFiles = []
653  for i in range(3):
654  copiedFiles = cp(archiveDir,workingDir,selectedFilesToProcess)
655  if len(copiedFiles) == len(selectedFilesToProcess):
656  break;
657  else:
658  subprocess.getstatusoutput("rm -rf " + workingDir)
659  if len(copiedFiles) != len(selectedFilesToProcess):
660  error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(selectedFilesToProcess)) + " from " + archiveDir + " to " + workingDir
661  sendEmail(mailList,error)
662  exit(error)
663 
664  print("Sorting and cleaning beamlist")
665  beamSpotObjList = []
666  for fileName in copiedFiles:
667  readBeamSpotFile(workingDir+fileName,beamSpotObjList,fileIOVBase)
668 
669  sortAndCleanBeamList(beamSpotObjList,fileIOVBase)
670 
671  if len(beamSpotObjList) == 0:
672  error = "WARNING: None of the processed and copied payloads has a valid fit so there are no results. This shouldn't happen since we are filtering using the run register, so there should be at least one good run."
673  exit(error)
674 
675  payloadFileName = "PayloadFile.txt"
676 
677  runBased = False
678  if dbIOVBase == "runnumber":
679  runBased = True
680 
681  payloadList = createWeightedPayloads(workingDir+payloadFileName,beamSpotObjList,runBased)
682  if len(payloadList) == 0:
683  error = "WARNING: I wasn't able to create any payload even if I have some BeamSpot objects."
684  exit(error)
685 
686 
687  tmpPayloadFileName = workingDir + "SingleTmpPayloadFile.txt"
688  tmpSqliteFileName = workingDir + "SingleTmpSqliteFile.db"
689 
690  writeDBTemplate = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/test/write2DB_template.py"
691  readDBTemplate = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/test/readDB_template.py"
692  payloadNumber = -1
693  iovSinceFirst = '0';
694  iovTillLast = '0';
695 
696  #Creating the final name for the combined sqlite file
697  uuid = subprocess.getstatusoutput('uuidgen -t')[1]
698  final_sqlite_file_name = databaseTag + '@' + uuid
699  sqlite_file = workingDir + final_sqlite_file_name + ".db"
700  metadata_file = workingDir + final_sqlite_file_name + ".txt"
701 
702  for payload in payloadList:
703  payloadNumber += 1
704  if option.zlarge:
705  payload.sigmaZ = 10
706  payload.sigmaZerr = 2.5e-05
707  tmpFile = file(tmpPayloadFileName,'w')
708  dumpValues(payload,tmpFile)
709  tmpFile.close()
710  if not writeSqliteFile(tmpSqliteFileName,databaseTag,dbIOVBase,tmpPayloadFileName,writeDBTemplate,workingDir):
711  error = "An error occurred while writing the sqlite file: " + tmpSqliteFileName
712  exit(error)
713  readSqliteFile(tmpSqliteFileName,databaseTag,readDBTemplate,workingDir)
714 
715  ##############################################################
716  #WARNING I am not sure if I am packing the right values
717  if dbIOVBase == "runnumber":
718  iov_since = str(payload.Run)
719  iov_till = iov_since
720  elif dbIOVBase == "lumiid":
721  iov_since = str( pack(int(payload.Run), int(payload.IOVfirst)) )
722  iov_till = str( pack(int(payload.Run), int(payload.IOVlast)) )
723  elif dbIOVBase == "timestamp":
724  error = "ERROR: IOV " + dbIOVBase + " still not implemented."
725  exit(error)
726  else:
727  error = "ERROR: IOV " + dbIOVBase + " unrecognized!"
728  exit(error)
729 
730  if payloadNumber == 0:
731  iovSinceFirst = iov_since
732  if payloadNumber == len(payloadList)-1:
733  iovTillLast = iov_till
734 
735  appendSqliteFile(final_sqlite_file_name + ".db", tmpSqliteFileName, databaseTag, iov_since, iov_till ,workingDir)
736  os.system("rm -f " + tmpPayloadFileName + " " + tmpSqliteFileName)
737 
738 
739  #### CREATE payload for merged output
740 
741  print(" create MERGED payload card for dropbox ...")
742 
743  dfile = open(metadata_file,'w')
744 
745  dfile.write('destDB ' + destDB +'\n')
746  dfile.write('tag ' + databaseTag +'\n')
747  dfile.write('inputtag' +'\n')
748  dfile.write('since ' + iovSinceFirst +'\n')
749  #dfile.write('till ' + iov_till +'\n')
750  dfile.write('Timetype '+ dbIOVBase +'\n')
751 
752  ###################################################
753  # WARNING tagType forced to offline
754  print("WARNING TAG TYPE forced to be just offline")
755  tagType = "offline"
756  checkType = tagType
757  if tagType == "express":
758  checkType = "hlt"
759  dfile.write('IOVCheck ' + checkType + '\n')
760  dfile.write('usertext Beam spot position\n')
761 
762  dfile.close()
763 
764 
765 
766  if option.upload:
767  print(" scp files to offline Drop Box")
768  dropbox = "/DropBox"
769  if option.Test:
770  dropbox = "/DropBox_test"
771  print("UPLOADING TO TEST DB")
772  uploadSqliteFile(workingDir, final_sqlite_file_name, dropbox)
773 
774  archive_sqlite_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + final_sqlite_file_name
775  archive_results_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + databaseTag + ".txt"
776  if not os.path.isdir(archiveDir + 'payloads'):
777  os.mkdir(archiveDir + 'payloads')
778  subprocess.getstatusoutput('mv ' + sqlite_file + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.db')
779  subprocess.getstatusoutput('mv ' + metadata_file + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.txt')
780  subprocess.getstatusoutput('cp ' + workingDir + payloadFileName + ' ' + archiveDir + 'payloads/' + archive_results_file_name)
781 
782  print(archiveDir + "payloads/" + archive_sqlite_file_name + '.db')
783  print(archiveDir + "payloads/" + archive_sqlite_file_name + '.txt')
784 
785  rmLock()
CommonMethods.sendEmail
def sendEmail
General utilities.
Definition: CommonMethods.py:150
CommonMethods.createWeightedPayloads
def createWeightedPayloads
CREATE FILE FOR PAYLOADS.
Definition: CommonMethods.py:624
hgcal_conditions::parameters
Definition: HGCConditions.h:86
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
if
if(conf_.getParameter< bool >("UseStripCablingDB"))
Definition: SiStripLAProfileBooker.cc:72
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
trklet::dirExists
int dirExists(const std::string &path)
Definition: Util.h:121
BeamSpotWorkflow.getLastUploadedIOV
def getLastUploadedIOV
General functions.
Definition: BeamSpotWorkflow.py:58
CommonMethods.sortAndCleanBeamList
def sortAndCleanBeamList
Sort and clean list of data for consecutive duplicates and bad fits.
Definition: CommonMethods.py:592
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.removeUncompleteRuns (   newRunList,
  dataSet 
)

Definition at line 475 of file BeamSpotWorkflow.py.

References getNumberOfFilesToProcessForRun(), getRunNumberFromFileName(), print(), and str.

475 
476 def removeUncompleteRuns(newRunList,dataSet):
477  processedRuns = {}
478  for fileName in newRunList:
479  run = getRunNumberFromFileName(fileName)
480  if not run in processedRuns:
481  processedRuns[run] = 0
482  processedRuns[run] += 1
483 
484  for run in processedRuns.keys():
485  nFiles = getNumberOfFilesToProcessForRun(dataSet,run)
486  if processedRuns[run] < nFiles:
487  print("I haven't processed all files yet : " + str(processedRuns[run]) + " out of " + str(nFiles) + " for run: " + str(run))
488  else:
489  print("All files have been processed for run: " + str(run) + " (" + str(processedRuns[run]) + " out of " + str(nFiles) + ")")
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
BeamSpotWorkflow.getNumberOfFilesToProcessForRun
def getNumberOfFilesToProcessForRun
Definition: BeamSpotWorkflow.py:92
str
#define str(s)
Definition: TestProcessor.cc:53
def BeamSpotWorkflow.selectFilesToProcess (   listOfRunsAndLumiFromDBS,
  listOfRunsAndLumiFromRR,
  newRunList,
  runListDir,
  dataSet,
  mailList,
  dbsTolerance,
  dbsTolerancePercent,
  rrTolerance,
  missingFilesTolerance,
  missingLumisTimeout 
)

Definition at line 259 of file BeamSpotWorkflow.py.

References bitset_utilities.append(), compareLumiLists(), beamvalidation.exit(), getNumberOfFilesToProcessForRun(), print(), sistrip::SpyUtilities.range(), python.rootplot.root2matplotlib.replace(), CommonMethods.sendEmail(), submitPVValidationJobs.split(), str, and CommonMethods.timeoutManager().

Referenced by main().

259 
260 def selectFilesToProcess(listOfRunsAndLumiFromDBS,listOfRunsAndLumiFromRR,newRunList,runListDir,dataSet,mailList,dbsTolerance,dbsTolerancePercent,rrTolerance,missingFilesTolerance,missingLumisTimeout):
261  runsAndLumisProcessed = {}
262  runsAndFiles = {}
263  for fileName in newRunList:
264  file = open(runListDir+fileName)
265  for line in file:
266  if line.find("Runnumber") != -1:
267  run = long(line.replace('\n','').split(' ')[1])
268  elif line.find("LumiRange") != -1:
269  lumiLine = line.replace('\n','').split(' ')
270  begLumi = long(lumiLine[1])
271  endLumi = long(lumiLine[3])
272  if begLumi != endLumi:
273  error = "The lumi range is greater than 1 for run " + str(run) + " " + line + " in file: " + runListDir + fileName
274  exit(error)
275  else:
276  if not run in runsAndLumisProcessed:
277  runsAndLumisProcessed[run] = []
278  if begLumi in runsAndLumisProcessed[run]:
279  print("Lumi " + str(begLumi) + " in event " + str(run) + " already exist. This MUST not happen but right now I will ignore this lumi!")
280  else:
281  runsAndLumisProcessed[run].append(begLumi)
282  if not run in runsAndFiles:
283  runsAndFiles[run] = []
284  runsAndFiles[run].append(fileName)
285  file.close()
286 
287  rrKeys = sorted(listOfRunsAndLumiFromRR.keys())
288  dbsKeys = listOfRunsAndLumiFromDBS.keys()
289  dbsKeys.sort()
290  #I remove the last entry from DBS since I am not sure it is an already closed run!
291  lastUnclosedRun = dbsKeys.pop()
292  #print "Last unclosed run: " + str(lastUnclosedRun)
293  procKeys = runsAndLumisProcessed.keys()
294  procKeys.sort()
295  #print "Run Registry:"
296  #print rrKeys
297  #print "DBS:"
298  #print dbsKeys
299  #print "List:"
300  #print procKeys
301  #print lastUnclosedRun
302  filesToProcess = []
303  for run in rrKeys:
304  RRList = []
305  for lumiRange in listOfRunsAndLumiFromRR[run]:
306  if lumiRange != []:
307  for l in range(lumiRange[0],lumiRange[1]+1):
308  RRList.append(long(l))
309  if run in procKeys and run < lastUnclosedRun:
310  #print "run " + str(run) + " is in procKeys"
311  if not run in dbsKeys and run != lastUnclosedRun:
312  error = "Impossible but run " + str(run) + " has been processed and it is also in the run registry but it is not in DBS!"
313  exit(error)
314  print("Working on run " + str(run))
315  nFiles = 0
316  for data in dataSet.split(','):
317  nFiles = getNumberOfFilesToProcessForRun(data,run)
318  if nFiles != 0:
319  break
320  if len(runsAndFiles[run]) < nFiles:
321  print("I haven't processed all files yet : " + str(len(runsAndFiles[run])) + " out of " + str(nFiles) + " for run: " + str(run))
322  if nFiles - len(runsAndFiles[run]) <= missingFilesTolerance:
323  timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run)) # resetting this timeout
324  timeoutType = timeoutManager("DBS_MISMATCH_Run"+str(run),missingLumisTimeout)
325  if timeoutType == 1:
326  print("WARNING: I previously set a timeout that expired...I'll continue with the script even if I didn't process all the lumis!")
327  else:
328  if timeoutType == -1:
329  print("WARNING: Setting the DBS_MISMATCH_Run" + str(run) + " timeout because I haven't processed all files!")
330  else:
331  print("WARNING: Timeout DBS_MISMATCH_Run" + str(run) + " is in progress.")
332  return filesToProcess
333  else:
334  timeoutType = timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run),missingLumisTimeout)
335  if timeoutType == 1:
336  error = "ERROR: I previously set a timeout that expired...I can't continue with the script because there are too many (" + str(nFiles - len(runsAndFiles[run])) + " files missing) and for too long " + str(missingLumisTimeout/3600) + " hours! I will process anyway the runs before this one (" + str(run) + ")"
337  sendEmail(mailList,error)
338  return filesToProcess
339  #exit(error)
340  else:
341  if timeoutType == -1:
342  print("WARNING: Setting the DBS_VERY_BIG_MISMATCH_Run" + str(run) + " timeout because I haven't processed all files!")
343  else:
344  print("WARNING: Timeout DBS_VERY_BIG_MISMATCH_Run" + str(run) + " is in progress.")
345  return filesToProcess
346 
347  else:
348  timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run))
349  timeoutManager("DBS_MISMATCH_Run"+str(run))
350  print("I have processed " + str(len(runsAndFiles[run])) + " out of " + str(nFiles) + " files that are in DBS. So I should have all the lumis!")
351  errors = []
352  badProcessed = []
353  badDBSProcessed = []
354  badDBS = []
355  badRRProcessed = []
356  badRR = []
357  #It is important for runsAndLumisProcessed[run] to be the first because the comparision is not ==
358  badDBSProcessed,badDBS = compareLumiLists(runsAndLumisProcessed[run],listOfRunsAndLumiFromDBS[run],errors)
359  for i in range(0,len(errors)):
360  errors[i] = errors[i].replace("listA","the processed lumis")
361  errors[i] = errors[i].replace("listB","DBS")
362  #print errors
363  #print badProcessed
364  #print badDBS
365  #exit("ciao")
366  if len(badDBS) != 0:
367  print("This is weird because I processed more lumis than the ones that are in DBS!")
368  if len(badDBSProcessed) != 0 and run in rrKeys:
369  lastError = len(errors)
370  #print RRList
371  #It is important for runsAndLumisProcessed[run] to be the first because the comparision is not ==
372  badRRProcessed,badRR = compareLumiLists(runsAndLumisProcessed[run],RRList,errors)
373  for i in range(0,len(errors)):
374  errors[i] = errors[i].replace("listA","the processed lumis")
375  errors[i] = errors[i].replace("listB","Run Registry")
376  #print errors
377  #print badProcessed
378  #print badRunRegistry
379 
380  if len(badRRProcessed) != 0:
381  print("I have not processed some of the lumis that are in the run registry for run: " + str(run))
382  for lumi in badDBSProcessed:
383  if lumi in badRRProcessed:
384  badProcessed.append(lumi)
385  lenA = len(badProcessed)
386  lenB = len(RRList)
387  if 100.*lenA/lenB <= dbsTolerancePercent:
388  print("WARNING: I didn't process " + str(100.*lenA/lenB) + "% of the lumis but I am within the " + str(dbsTolerancePercent) + "% set in the configuration. Which corrispond to " + str(lenA) + " out of " + str(lenB) + " lumis")
389  #print errors
390  badProcessed = []
391  elif lenA <= dbsTolerance:
392  print("WARNING: I didn't process " + str(lenA) + " lumis but I am within the " + str(dbsTolerance) + " lumis set in the configuration. Which corrispond to " + str(lenA) + " out of " + str(lenB) + " lumis")
393  #print errors
394  badProcessed = []
395  else:
396  error = "ERROR: For run " + str(run) + " I didn't process " + str(100.*lenA/lenB) + "% of the lumis and I am not within the " + str(dbsTolerancePercent) + "% set in the configuration. The number of lumis that I didn't process (" + str(lenA) + " out of " + str(lenB) + ") is greater also than the " + str(dbsTolerance) + " lumis that I can tolerate. I can't process runs >= " + str(run) + " but I'll process the runs before!"
397  sendEmail(mailList,error)
398  print(error)
399  return filesToProcess
400  #exit(errors)
401  #return filesToProcess
402  elif len(errors) != 0:
403  print("The number of lumi sections processed didn't match the one in DBS but they cover all the ones in the Run Registry, so it is ok!")
404  #print errors
405 
406  #If I get here it means that I passed or the DBS or the RR test
407  if len(badProcessed) == 0:
408  for file in runsAndFiles[run]:
409  filesToProcess.append(file)
410  else:
411  #print errors
412  print("This should never happen because if I have errors I return or exit! Run: " + str(run))
413  else:
414  error = "Run " + str(run) + " is in the run registry but it has not been processed yet!"
415  print(error)
416  timeoutType = timeoutManager("MISSING_RUNREGRUN_Run"+str(run),missingLumisTimeout)
417  if timeoutType == 1:
418  if len(RRList) <= rrTolerance:
419  error = "WARNING: I previously set the MISSING_RUNREGRUN_Run" + str(run) + " timeout that expired...I am missing run " + str(run) + " but it only had " + str(len(RRList)) + " <= " + str(rrTolerance) + " lumis. So I will continue and ignore it... "
420  #print listOfRunsAndLumiFromRR[run]
421  print(error)
422  #sendEmail(mailList,error)
423  else:
424  error = "ERROR: I previously set the MISSING_RUNREGRUN_Run" + str(run) + " timeout that expired...I am missing run " + str(run) + " which has " + str(len(RRList)) + " > " + str(rrTolerance) + " lumis. I can't continue but I'll process the runs before this one"
425  sendEmail(mailList,error)
426  return filesToProcess
427  #exit(error)
428  else:
429  if timeoutType == -1:
430  print("WARNING: Setting the MISSING_RUNREGRUN_Run" + str(run) + " timeout because I haven't processed a run!")
431  else:
432  print("WARNING: Timeout MISSING_RUNREGRUN_Run" + str(run) + " is in progress.")
433  return filesToProcess
434 
return filesToProcess
CommonMethods.sendEmail
def sendEmail
General utilities.
Definition: CommonMethods.py:150
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
BeamSpotWorkflow.getNumberOfFilesToProcessForRun
def getNumberOfFilesToProcessForRun
Definition: BeamSpotWorkflow.py:92
str
#define str(s)
Definition: TestProcessor.cc:53

Variable Documentation

string BeamSpotWorkflow.error = "Please set a crab environment in order to get the proper JSON lib"

Definition at line 52 of file BeamSpotWorkflow.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5