4 Handle lists of lumi sections. Constuct in several different formats and filter
5 (mask) a secondary list of lumis.
6 This class can also handle ranges of events as the structure is identical
7 or could be subclassed renaming a function or two.
9 This code began life in COMP/CRAB/python/LumiList.py
20 Deal with lists of lumis in several different forms:
23 '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
26 where the first key is the run number, subsequent pairs are
27 ranges of lumis within that run that are desired
30 '1': [1,2,3,4,6,7,8,9,10],
33 where the first key is the run number and the list is a list of
34 individual lumi sections. This form also takes a list of these objects
35 which can be much faster than LumiList += LumiList
37 [[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]]
38 where each pair in the list is an individual run&lumi
40 '1:1-1:33,1:35,1:37-1:47,2:1-2:45,2:50-2:80'
41 The string used by CMSSW in lumisToProcess or lumisToSkip
42 is a subset of the compactList example above
46 def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None, url = None):
48 Constructor takes filename (JSON), a list of run/lumi pairs,
49 or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs
59 jsonFile = urllib2.urlopen(url)
63 for (run, lumi)
in lumis:
65 if run
not in runsAndLumis:
66 runsAndLumis[run] = []
67 runsAndLumis[run].
append(lumi)
69 if isinstance(runsAndLumis, list):
71 for runLumiList
in runsAndLumis:
72 for run, lumis
in runLumiList.items():
73 queued.setdefault(run, []).extend(lumis)
77 for run
in runsAndLumis.keys():
80 lumiList = runsAndLumis[run]
84 for lumi
in sorted(lumiList):
87 elif lumi != lastLumi + 1:
99 for run
in compactList.keys():
106 for run
in self.compactList.keys():
110 if newLumis
and newLumis[-1][0] <= lumi[0] <= newLumis[-1][1] + 1:
111 newLumis[-1][1] =
max(newLumis[-1][1], lumi[1])
113 newLumis.append(lumi)
118 for run
in sorted(self.compactList.keys()):
120 blumis = sorted(other.compactList.get(run, []))
123 tmplist = [alumi[0], alumi[1]]
125 if blumi[0] <= tmplist[0]
and blumi[1] >= tmplist[1]:
128 if blumi[0] > tmplist[0]
and blumi[1] < tmplist[1]:
129 alist.append([tmplist[0], blumi[0]-1])
130 tmplist = [blumi[1]+1, tmplist[1]]
131 elif blumi[0] <= tmplist[0]
and blumi[1] < tmplist[1]
and blumi[1]>=tmplist[0]:
132 tmplist = [blumi[1]+1, tmplist[1]]
133 elif blumi[0] > tmplist[0]
and blumi[1] >= tmplist[1]
and blumi[0]<=tmplist[1]:
134 alist.append([tmplist[0], blumi[0]-1])
138 alist.append(tmplist)
141 return LumiList(compactList = result)
146 aruns = set(self.compactList.keys())
147 bruns = set(other.compactList.keys())
148 for run
in aruns & bruns:
152 for blumi
in other.compactList[run]:
153 if blumi[0] <= alumi[0]
and blumi[1] >= alumi[1]:
154 lumiList.append(alumi)
155 if blumi[0] > alumi[0]
and blumi[1] < alumi[1]:
156 lumiList.append(blumi)
157 elif blumi[0] <= alumi[0]
and blumi[1] < alumi[1]
and blumi[1] >= alumi[0]:
158 lumiList.append([alumi[0], blumi[1]])
159 elif blumi[0] > alumi[0]
and blumi[1] >= alumi[1]
and blumi[0] <= alumi[1]:
160 lumiList.append([blumi[0], alumi[1]])
164 unique = [lumiList[0]]
165 for pair
in lumiList[1:]:
166 if pair[0] == unique[-1][1]+1:
167 unique[-1][1] = copy.deepcopy(pair[1])
169 unique.append(copy.deepcopy(pair))
172 return LumiList(compactList = result)
177 aruns = self.compactList.keys()
178 bruns = other.compactList.keys()
179 runs = set(aruns + bruns)
181 overlap = sorted(self.compactList.get(run, []) + other.compactList.get(run, []))
182 unique = [overlap[0]]
183 for pair
in overlap[1:]:
184 if pair[0] >= unique[-1][0]
and pair[0] <= unique[-1][1]+1
and pair[1] > unique[-1][1]:
185 unique[-1][1] = copy.deepcopy(pair[1])
186 elif pair[0] > unique[-1][1]:
187 unique.append(copy.deepcopy(pair))
189 return LumiList(compactList = result)
197 '''Returns number of runs in list'''
202 Return a list of lumis that are in compactList.
203 lumilist is of the simple form
204 [(run1,lumi1),(run1,lumi2),(run2,lumi1)]
207 for (run, lumi)
in lumiList:
208 runsInLumi = self.compactList.get(str(run), [[0, -1]])
209 for (first, last)
in runsInLumi:
210 if lumi >= first
and lumi <= last:
211 filteredList.append((run, lumi))
217 doubleBracketRE = re.compile (
r']],')
218 return doubleBracketRE.sub (
']],\n',
224 Return the compact list representation
231 Return the list of duplicates found during construction as a LumiList
238 Return the list of pairs representation
241 runs = self.compactList.keys()
245 for lumiPair
in sorted(lumis):
246 for lumi
in range(lumiPair[0], lumiPair[1]+1):
247 theList.append((int(run), lumi))
254 return the sorted list of runs contained
256 return sorted (self.compactList.keys())
261 Turn compactList into a list of the format
262 [ 'R1:L1', 'R2:L2-R2:L3' ] which is used by getCMSSWString and getVLuminosityBlockRange
266 runs = self.compactList.keys()
270 for lumiPair
in sorted(lumis):
271 if lumiPair[0] == lumiPair[1]:
272 parts.append(
"%s:%s" % (run, lumiPair[0]))
274 parts.append(
"%s:%s-%s:%s" %
275 (run, lumiPair[0], run, lumiPair[1]))
281 Turn compactList into a list of the format
282 R1:L1,R2:L2-R2:L3 which is acceptable to CMSSW LumiBlockRange variable
286 output =
','.
join(parts)
292 Turn compactList into an (optionally tracked) VLuminosityBlockRange
295 import FWCore.ParameterSet.Config
as cms
298 return cms.VLuminosityBlockRange(parts)
300 return cms.untracked.VLuminosityBlockRange(parts)
305 Write out a JSON file representation of the object
307 jsonFile = open(fileName,
'w')
308 jsonFile.write(
"%s\n" % self)
314 removes runs from runList from collection
326 Selects only runs from runList in collection
329 for run
in self.compactList.keys():
330 if int(run)
not in runList
and run
not in runList:
331 runsToDelete.append(run)
333 for run
in runsToDelete:
340 returns true if the run, lumi section passed in is contained
341 in this lumiList. Input can be either:
342 - a single tuple of (run, lumi),
343 - separate run and lumi numbers
344 - a single run number (returns true if any lumi sections exist)
346 if lumiSection
is None:
348 if isinstance (run, int)
or isinstance (run, str):
355 raise RuntimeError(
"Improper format for run '%s'" % run)
356 lumiRangeList = self.compactList.get( str(run) )
357 if not lumiRangeList:
360 for lumiRange
in lumiRangeList:
366 if lumiRange[0] <= lumiSection
and \
367 (0 == lumiRange[1]
or lumiSection <= lumiRange[1]):
381 import FWCore.ParameterSet.Config as cms
383 class LumiListTest(unittest.TestCase):
391 Test reading from JSON
393 exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
394 exDict = {'1': [[1, 33], [35, 35], [37, 47]],
395 '2': [[49, 75], [77, 130], [133, 136]]}
396 exVLBR = cms.VLuminosityBlockRange('1:1-1:33', '1:35', '1:37-1:47', '2:49-2:75', '2:77-2:130', '2:133-2:136')
398 jsonList = LumiList(filename = 'lumiTest.json')
399 lumiString = jsonList.getCMSSWString()
400 lumiList = jsonList.getCompactList()
401 lumiVLBR = jsonList.getVLuminosityBlockRange(True)
403 self.assertTrue(lumiString == exString)
404 self.assertTrue(lumiList == exDict)
405 self.assertTrue(lumiVLBR == exVLBR)
409 Test constucting from list of pairs
412 listLs1 = range(1, 34) + [35] + range(37, 48)
413 listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
414 lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2)
416 jsonLister = LumiList(filename = 'lumiTest.json')
417 jsonString = jsonLister.getCMSSWString()
418 jsonList = jsonLister.getCompactList()
420 pairLister = LumiList(lumis = lumis)
421 pairString = pairLister.getCMSSWString()
422 pairList = pairLister.getCompactList()
424 self.assertTrue(jsonString == pairString)
425 self.assertTrue(jsonList == pairList)
430 Test constucting from run and list of lumis
433 1: range(1, 34) + [35] + range(37, 48),
434 2: range(49, 76) + range(77, 131) + range(133, 137)
437 '1': range(1, 34) + [35] + range(37, 48),
438 '2': range(49, 76) + range(77, 131) + range(133, 137)
445 jsonLister = LumiList(filename = 'lumiTest.json')
446 jsonString = jsonLister.getCMSSWString()
447 jsonList = jsonLister.getCompactList()
449 runLister = LumiList(runsAndLumis = runsAndLumis)
450 runString = runLister.getCMSSWString()
451 runList = runLister.getCompactList()
453 runLister2 = LumiList(runsAndLumis = runsAndLumis2)
454 runList2 = runLister2.getCompactList()
456 runLister3 = LumiList(runsAndLumis = blank)
459 self.assertTrue(jsonString == runString)
460 self.assertTrue(jsonList == runList)
461 self.assertTrue(runList2 == runList)
462 self.assertTrue(len(runLister3) == 0)
464 def testFilter(self):
466 Test filtering of a list of lumis
469 1: range(1, 34) + [35] + range(37, 48),
470 2: range(49, 76) + range(77, 131) + range(133, 137)
473 completeList = zip([1]*150, range(1, 150)) + \
474 zip([2]*150, range(1, 150)) + \
475 zip([3]*150, range(1, 150))
477 smallList = zip([1]*50, range(1, 10)) + zip([2]*50, range(50, 70))
478 overlapList = zip([1]*150, range(30, 40)) + \
479 zip([2]*150, range(60, 80))
480 overlapRes = zip([1]*9, range(30, 34)) + [(1, 35)] + \
481 zip([1]*9, range(37, 40)) + \
482 zip([2]*30, range(60, 76)) + \
483 zip([2]*9, range(77, 80))
485 runLister = LumiList(runsAndLumis = runsAndLumis)
487 # Test a list to be filtered which is a superset of constructed list
488 filterComplete = runLister.filterLumis(completeList)
489 # Test a list to be filtered which is a subset of constructed list
490 filterSmall = runLister.filterLumis(smallList)
491 # Test a list to be filtered which is neither
492 filterOverlap = runLister.filterLumis(overlapList)
494 self.assertTrue(filterComplete == runLister.getLumis())
495 self.assertTrue(filterSmall == smallList)
496 self.assertTrue(filterOverlap == overlapRes)
498 def testDuplicates(self):
500 Test a list with lots of duplicates
502 result = zip([1]*100, range(1, 34) + range(37, 48))
503 lumis = zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25))
505 lister = LumiList(lumis = lumis)
506 self.assertTrue(lister.getLumis() == result)
513 runLister = LumiList(lumis = None)
515 self.assertTrue(runLister.getCMSSWString() == '')
516 self.assertTrue(runLister.getLumis() == [])
517 self.assertTrue(runLister.getCompactList() == {})
519 def testSubtract(self):
521 a-b for lots of cases
524 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
525 '2' : range(6,20) + range (30,40),
526 '3' : range(10,20) + range (30,40) + range(50,60),
528 blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(33,36),
530 '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
532 clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(33,36),
535 result = {'1' : range(6,12) + range(13,16) + range(31,33) + range(36,39),
536 '2' : range(6,10) + range(35,40),
537 '3' : range(15,20) + range(30,35) + range(51,59),
539 result2 = {'1' : range(6,12) + range(13,16) + range(31,33) + range(36,39),
540 '2' : range(6,10) + range(35,40),
541 '3' : range(10,20) + range (30,40) + range(50,60),
543 a = LumiList(runsAndLumis = alumis)
544 b = LumiList(runsAndLumis = blumis)
545 c = LumiList(runsAndLumis = clumis)
546 r = LumiList(runsAndLumis = result)
547 r2 = LumiList(runsAndLumis = result2)
549 self.assertTrue((a-b).getCMSSWString() == r.getCMSSWString())
550 self.assertTrue((a-b).getCMSSWString() != (b-a).getCMSSWString())
551 # Test where c is missing runs from a
552 self.assertTrue((a-c).getCMSSWString() == r2.getCMSSWString())
553 self.assertTrue((a-c).getCMSSWString() != (c-a).getCMSSWString())
555 self.assertTrue(str(a-a) == '{}')
556 self.assertTrue(len(a-a) == 0)
560 a|b for lots of cases
563 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
564 '2' : range(6,20) + range (30,40),
565 '3' : range(10,20) + range (30,40) + range(50,60),
567 blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
569 '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
571 clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
574 result = {'1' : range(2,20) + range(31,39) + range(45,49) + range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
575 '2' : range(6,20) + range (30,40) + range(10,35),
576 '3' : range(10,20) + range (30,40) + range(50,60) + range(10,15) + range(35,40) + range(45,51) + range(59,70),
578 a = LumiList(runsAndLumis = alumis)
579 b = LumiList(runsAndLumis = blumis)
580 c = LumiList(runsAndLumis = blumis)
581 r = LumiList(runsAndLumis = result)
582 self.assertTrue((a|b).getCMSSWString() == r.getCMSSWString())
583 self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
584 self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())
586 # Test list constuction (faster)
588 multiple = [alumis, blumis, clumis]
589 easy = LumiList(runsAndLumis = multiple)
592 self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())
596 a&b for lots of cases
599 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
600 '2' : range(6,20) + range (30,40),
601 '3' : range(10,20) + range (30,40) + range(50,60),
604 blumis = {'1' : range(1,6) + range(12,13) + range(16,25) + range(25,40) + range(40,50) + range(33,36),
606 '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
609 result = {'1' : range(2,6) + range(12,13) + range(16,20) + range(31,39) + range(45,49),
610 '2' : range(10,20) + range(30,35),
611 '3' : range(10,15) + range(35,40) + range(50,51)+ range(59,60),
613 a = LumiList(runsAndLumis = alumis)
614 b = LumiList(runsAndLumis = blumis)
615 r = LumiList(runsAndLumis = result)
616 self.assertTrue((a&b).getCMSSWString() == r.getCMSSWString())
617 self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
618 self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())
620 def testRemoveSelect(self):
622 a-b for lots of cases
625 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
626 '2' : range(6,20) + range (30,40),
627 '3' : range(10,20) + range (30,40) + range(50,60),
628 '4' : range(10,20) + range (30,80),
631 result = {'2' : range(6,20) + range (30,40),
632 '4' : range(10,20) + range (30,80),
635 rem = LumiList(runsAndLumis = alumis)
636 sel = LumiList(runsAndLumis = alumis)
637 res = LumiList(runsAndLumis = result)
639 rem.removeRuns([1,3])
640 sel.selectRuns([2,4])
642 self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
643 self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
644 self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())
647 URL = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions12/8TeV/Reprocessing/Cert_190456-195530_8TeV_08Jun2012ReReco_Collisions12_JSON.txt'
648 ll = LumiList(url=URL)
649 self.assertTrue(len(ll) > 0)
653 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
654 '2' : range(6,20) + range (30,40),
655 '3' : range(10,20) + range (30,40) + range(50,60),
658 a = LumiList(runsAndLumis = alumis)
659 a.writeJSON('newFile.json')
662 if __name__ == '__main__':
663 jsonFile = open('lumiTest.json','w')
664 jsonFile.write('{"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}')
672 if __name__ ==
'__main__':
679 historyPath = os.path.expanduser(
"~/.pyhistory")
684 readline.write_history_file(historyPath)
685 if os.path.exists(historyPath):
686 readline.read_history_file(historyPath)
689 atexit.register(save_history)
690 readline.parse_and_bind(
"set show-all-if-ambiguous on")
691 readline.parse_and_bind(
"tab: complete")
692 if os.path.exists (historyPath) :
693 readline.read_history_file(historyPath)
694 readline.set_history_length(-1)
static std::string join(char **cmd)
def getVLuminosityBlockRange