4 Handle lists of lumi sections. Constuct in several different formats and filter
5 (mask) a secondary list of lumis.
6 This class can also handle ranges of events as the structure is identical
7 or could be subclassed renaming a function or two.
9 This code began life in COMP/CRAB/python/LumiList.py
12 __revision__ =
"$Id: LumiList.py,v 1.15 2011/11/02 16:06:57 ewv Exp $"
13 __version__ =
"$Revision: 1.15 $"
22 Deal with lists of lumis in several different forms:
25 '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
28 where the first key is the run number, subsequent pairs are
29 ranges of lumis within that run that are desired
32 '1': [1,2,3,4,6,7,8,9,10],
35 where the first key is the run number and the list is a list of
36 individual lumi sections. This form also takes a list of these objects
37 which can be much faster than LumiList += LumiList
39 [[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]]
40 where each pair in the list is an individual run&lumi
42 '1:1-1:33,1:35,1:37-1:47,2:1-2:45,2:50-2:80'
43 The string used by CMSSW in lumisToProcess or lumisToSkip
44 is a subset of the compactList example above
48 def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None, compactList = None, url = None):
50 Constructor takes filename (JSON), a list of run/lumi pairs,
51 or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs
61 jsonFile = urllib2.urlopen(url)
65 for (run, lumi)
in lumis:
67 if not runsAndLumis.has_key(run):
68 runsAndLumis[run] = []
69 runsAndLumis[run].
append(lumi)
71 if isinstance(runsAndLumis, list):
73 for runLumiList
in runsAndLumis:
74 for run, lumis
in runLumiList.items():
75 queued.setdefault(run, []).extend(lumis)
79 for run
in runsAndLumis.keys():
82 lumiList = runsAndLumis[run]
86 for lumi
in sorted(lumiList):
89 elif lumi != lastLumi + 1:
101 for run
in compactList.keys():
109 for run
in sorted(self.compactList.keys()):
111 blumis = sorted(other.compactList.get(run, []))
114 tmplist = [alumi[0], alumi[1]]
116 if blumi[0] <= tmplist[0]
and blumi[1] >= tmplist[1]:
119 if blumi[0] > tmplist[0]
and blumi[1] < tmplist[1]:
120 alist.append([tmplist[0], blumi[0]-1])
121 tmplist = [blumi[1]+1, tmplist[1]]
122 elif blumi[0] <= tmplist[0]
and blumi[1] < tmplist[1]
and blumi[1]>=tmplist[0]:
123 tmplist = [blumi[1]+1, tmplist[1]]
124 elif blumi[0] > tmplist[0]
and blumi[1] >= tmplist[1]
and blumi[0]<=tmplist[1]:
125 alist.append([tmplist[0], blumi[0]-1])
129 alist.append(tmplist)
132 return LumiList(compactList = result)
137 aruns =
set(self.compactList.keys())
138 bruns =
set(other.compactList.keys())
139 for run
in aruns & bruns:
143 for blumi
in other.compactList[run]:
144 if blumi[0] <= alumi[0]
and blumi[1] >= alumi[1]:
145 lumiList.append(alumi)
146 if blumi[0] > alumi[0]
and blumi[1] < alumi[1]:
147 lumiList.append(blumi)
148 elif blumi[0] <= alumi[0]
and blumi[1] < alumi[1]
and blumi[1] >= alumi[0]:
149 lumiList.append([alumi[0], blumi[1]])
150 elif blumi[0] > alumi[0]
and blumi[1] >= alumi[1]
and blumi[0] <= alumi[1]:
151 lumiList.append([blumi[0], alumi[1]])
155 unique = [lumiList[0]]
156 for pair
in lumiList[1:]:
157 if pair[0] == unique[-1][1]+1:
158 unique[-1][1] = copy.deepcopy(pair[1])
160 unique.append(copy.deepcopy(pair))
163 return LumiList(compactList = result)
168 aruns = self.compactList.keys()
169 bruns = other.compactList.keys()
170 runs =
set(aruns + bruns)
172 overlap = sorted(self.compactList.get(run, []) + other.compactList.get(run, []))
173 unique = [overlap[0]]
174 for pair
in overlap[1:]:
175 if pair[0] >= unique[-1][0]
and pair[0] <= unique[-1][1]+1
and pair[1] > unique[-1][1]:
176 unique[-1][1] = copy.deepcopy(pair[1])
177 elif pair[0] > unique[-1][1]:
178 unique.append(copy.deepcopy(pair))
180 return LumiList(compactList = result)
188 '''Returns number of runs in list'''
193 Return a list of lumis that are in compactList.
194 lumilist is of the simple form
195 [(run1,lumi1),(run1,lumi2),(run2,lumi1)]
198 for (run, lumi)
in lumiList:
199 runsInLumi = self.compactList.get(str(run), [[0, -1]])
200 for (first, last)
in runsInLumi:
201 if lumi >= first
and lumi <= last:
202 filteredList.append((run, lumi))
208 doubleBracketRE = re.compile (
r']],')
209 return doubleBracketRE.sub (
']],\n',
215 Return the compact list representation
222 Return the list of duplicates found during construction as a LumiList
229 Return the list of pairs representation
232 runs = self.compactList.keys()
236 for lumiPair
in sorted(lumis):
237 for lumi
in range(lumiPair[0], lumiPair[1]+1):
238 theList.append((int(run), lumi))
245 return the sorted list of runs contained
247 return sorted (self.compactList.keys())
252 Turn compactList into a list of the format
253 [ 'R1:L1', 'R2:L2-R2:L3' ] which is used by getCMSSWString and getVLuminosityBlockRange
257 runs = self.compactList.keys()
261 for lumiPair
in sorted(lumis):
262 if lumiPair[0] == lumiPair[1]:
263 parts.append(
"%s:%s" % (run, lumiPair[0]))
265 parts.append(
"%s:%s-%s:%s" %
266 (run, lumiPair[0], run, lumiPair[1]))
272 Turn compactList into a list of the format
273 R1:L1,R2:L2-R2:L3 which is acceptable to CMSSW LumiBlockRange variable
277 output =
','.
join(parts)
283 Turn compactList into an (optionally tracked) VLuminosityBlockRange
289 return cms.VLuminosityBlockRange(parts)
291 return cms.untracked.VLuminosityBlockRange(parts)
296 Write out a JSON file representation of the object
298 jsonFile = open(fileName,
'w')
299 jsonFile.write(
"%s\n" % self)
305 removes runs from runList from collection
309 if self.compactList.has_key (run):
317 Selects only runs from runList in collection
320 for run
in self.compactList.keys():
321 if int(run)
not in runList
and run
not in runList:
322 runsToDelete.append(run)
324 for run
in runsToDelete:
331 returns true if the run, lumi section passed in is contained
332 in this lumiList. Input can be either:
333 - a single tuple of (run, lumi),
334 - separate run and lumi numbers
335 - a single run number (returns true if any lumi sections exist)
337 if lumiSection
is None:
339 if isinstance (run, int)
or isinstance (run, str):
340 return self.compactList.has_key( str(run) )
346 raise RuntimeError,
"Improper format for run '%s'" % run
347 lumiRangeList = self.compactList.get( str(run) )
348 if not lumiRangeList:
351 for lumiRange
in lumiRangeList:
357 if lumiRange[0] <= lumiSection
and \
358 (0 == lumiRange[1]
or lumiSection <= lumiRange[1]):
372 import FWCore.ParameterSet.Config as cms
374 class LumiListTest(unittest.TestCase):
382 Test reading from JSON
384 exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
385 exDict = {'1': [[1, 33], [35, 35], [37, 47]],
386 '2': [[49, 75], [77, 130], [133, 136]]}
387 exVLBR = cms.VLuminosityBlockRange('1:1-1:33', '1:35', '1:37-1:47', '2:49-2:75', '2:77-2:130', '2:133-2:136')
389 jsonList = LumiList(filename = 'lumiTest.json')
390 lumiString = jsonList.getCMSSWString()
391 lumiList = jsonList.getCompactList()
392 lumiVLBR = jsonList.getVLuminosityBlockRange(True)
394 self.assertTrue(lumiString == exString)
395 self.assertTrue(lumiList == exDict)
396 self.assertTrue(lumiVLBR == exVLBR)
400 Test constucting from list of pairs
403 listLs1 = range(1, 34) + [35] + range(37, 48)
404 listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
405 lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2)
407 jsonLister = LumiList(filename = 'lumiTest.json')
408 jsonString = jsonLister.getCMSSWString()
409 jsonList = jsonLister.getCompactList()
411 pairLister = LumiList(lumis = lumis)
412 pairString = pairLister.getCMSSWString()
413 pairList = pairLister.getCompactList()
415 self.assertTrue(jsonString == pairString)
416 self.assertTrue(jsonList == pairList)
421 Test constucting from run and list of lumis
424 1: range(1, 34) + [35] + range(37, 48),
425 2: range(49, 76) + range(77, 131) + range(133, 137)
428 '1': range(1, 34) + [35] + range(37, 48),
429 '2': range(49, 76) + range(77, 131) + range(133, 137)
436 jsonLister = LumiList(filename = 'lumiTest.json')
437 jsonString = jsonLister.getCMSSWString()
438 jsonList = jsonLister.getCompactList()
440 runLister = LumiList(runsAndLumis = runsAndLumis)
441 runString = runLister.getCMSSWString()
442 runList = runLister.getCompactList()
444 runLister2 = LumiList(runsAndLumis = runsAndLumis2)
445 runList2 = runLister2.getCompactList()
447 runLister3 = LumiList(runsAndLumis = blank)
450 self.assertTrue(jsonString == runString)
451 self.assertTrue(jsonList == runList)
452 self.assertTrue(runList2 == runList)
453 self.assertTrue(len(runLister3) == 0)
455 def testFilter(self):
457 Test filtering of a list of lumis
460 1: range(1, 34) + [35] + range(37, 48),
461 2: range(49, 76) + range(77, 131) + range(133, 137)
464 completeList = zip([1]*150, range(1, 150)) + \
465 zip([2]*150, range(1, 150)) + \
466 zip([3]*150, range(1, 150))
468 smallList = zip([1]*50, range(1, 10)) + zip([2]*50, range(50, 70))
469 overlapList = zip([1]*150, range(30, 40)) + \
470 zip([2]*150, range(60, 80))
471 overlapRes = zip([1]*9, range(30, 34)) + [(1, 35)] + \
472 zip([1]*9, range(37, 40)) + \
473 zip([2]*30, range(60, 76)) + \
474 zip([2]*9, range(77, 80))
476 runLister = LumiList(runsAndLumis = runsAndLumis)
478 # Test a list to be filtered which is a superset of constructed list
479 filterComplete = runLister.filterLumis(completeList)
480 # Test a list to be filtered which is a subset of constructed list
481 filterSmall = runLister.filterLumis(smallList)
482 # Test a list to be filtered which is neither
483 filterOverlap = runLister.filterLumis(overlapList)
485 self.assertTrue(filterComplete == runLister.getLumis())
486 self.assertTrue(filterSmall == smallList)
487 self.assertTrue(filterOverlap == overlapRes)
489 def testDuplicates(self):
491 Test a list with lots of duplicates
493 result = zip([1]*100, range(1, 34) + range(37, 48))
494 lumis = zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25))
496 lister = LumiList(lumis = lumis)
497 self.assertTrue(lister.getLumis() == result)
504 runLister = LumiList(lumis = None)
506 self.assertTrue(runLister.getCMSSWString() == '')
507 self.assertTrue(runLister.getLumis() == [])
508 self.assertTrue(runLister.getCompactList() == {})
510 def testSubtract(self):
512 a-b for lots of cases
515 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
516 '2' : range(6,20) + range (30,40),
517 '3' : range(10,20) + range (30,40) + range(50,60),
519 blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(33,36),
521 '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
523 clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(33,36),
526 result = {'1' : range(6,12) + range(13,16) + range(31,33) + range(36,39),
527 '2' : range(6,10) + range(35,40),
528 '3' : range(15,20) + range(30,35) + range(51,59),
530 result2 = {'1' : range(6,12) + range(13,16) + range(31,33) + range(36,39),
531 '2' : range(6,10) + range(35,40),
532 '3' : range(10,20) + range (30,40) + range(50,60),
534 a = LumiList(runsAndLumis = alumis)
535 b = LumiList(runsAndLumis = blumis)
536 c = LumiList(runsAndLumis = clumis)
537 r = LumiList(runsAndLumis = result)
538 r2 = LumiList(runsAndLumis = result2)
540 self.assertTrue((a-b).getCMSSWString() == r.getCMSSWString())
541 self.assertTrue((a-b).getCMSSWString() != (b-a).getCMSSWString())
542 # Test where c is missing runs from a
543 self.assertTrue((a-c).getCMSSWString() == r2.getCMSSWString())
544 self.assertTrue((a-c).getCMSSWString() != (c-a).getCMSSWString())
546 self.assertTrue(str(a-a) == '{}')
547 self.assertTrue(len(a-a) == 0)
551 a|b for lots of cases
554 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
555 '2' : range(6,20) + range (30,40),
556 '3' : range(10,20) + range (30,40) + range(50,60),
558 blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
560 '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
562 clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
565 result = {'1' : range(2,20) + range(31,39) + range(45,49) + range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
566 '2' : range(6,20) + range (30,40) + range(10,35),
567 '3' : range(10,20) + range (30,40) + range(50,60) + range(10,15) + range(35,40) + range(45,51) + range(59,70),
569 a = LumiList(runsAndLumis = alumis)
570 b = LumiList(runsAndLumis = blumis)
571 c = LumiList(runsAndLumis = blumis)
572 r = LumiList(runsAndLumis = result)
573 self.assertTrue((a|b).getCMSSWString() == r.getCMSSWString())
574 self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
575 self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())
577 # Test list constuction (faster)
579 multiple = [alumis, blumis, clumis]
580 easy = LumiList(runsAndLumis = multiple)
583 self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())
587 a&b for lots of cases
590 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
591 '2' : range(6,20) + range (30,40),
592 '3' : range(10,20) + range (30,40) + range(50,60),
595 blumis = {'1' : range(1,6) + range(12,13) + range(16,25) + range(25,40) + range(40,50) + range(33,36),
597 '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
600 result = {'1' : range(2,6) + range(12,13) + range(16,20) + range(31,39) + range(45,49),
601 '2' : range(10,20) + range(30,35),
602 '3' : range(10,15) + range(35,40) + range(50,51)+ range(59,60),
604 a = LumiList(runsAndLumis = alumis)
605 b = LumiList(runsAndLumis = blumis)
606 r = LumiList(runsAndLumis = result)
607 self.assertTrue((a&b).getCMSSWString() == r.getCMSSWString())
608 self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
609 self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())
611 def testRemoveSelect(self):
613 a-b for lots of cases
616 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
617 '2' : range(6,20) + range (30,40),
618 '3' : range(10,20) + range (30,40) + range(50,60),
619 '4' : range(10,20) + range (30,80),
622 result = {'2' : range(6,20) + range (30,40),
623 '4' : range(10,20) + range (30,80),
626 rem = LumiList(runsAndLumis = alumis)
627 sel = LumiList(runsAndLumis = alumis)
628 res = LumiList(runsAndLumis = result)
630 rem.removeRuns([1,3])
631 sel.selectRuns([2,4])
633 self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
634 self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
635 self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())
638 URL = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions12/8TeV/Reprocessing/Cert_190456-195530_8TeV_08Jun2012ReReco_Collisions12_JSON.txt'
639 ll = LumiList(url=URL)
640 self.assertTrue(len(ll) > 0)
644 alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
645 '2' : range(6,20) + range (30,40),
646 '3' : range(10,20) + range (30,40) + range(50,60),
649 a = LumiList(runsAndLumis = alumis)
650 a.writeJSON('newFile.json')
653 if __name__ == '__main__':
654 jsonFile = open('lumiTest.json','w')
655 jsonFile.write('{"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}')
663 if __name__ ==
'__main__':
670 historyPath = os.path.expanduser(
"~/.pyhistory")
675 readline.write_history_file(historyPath)
676 if os.path.exists(historyPath):
677 readline.read_history_file(historyPath)
680 atexit.register(save_history)
681 readline.parse_and_bind(
"set show-all-if-ambiguous on")
682 readline.parse_and_bind(
"tab: complete")
683 if os.path.exists (historyPath) :
684 readline.read_history_file(historyPath)
685 readline.set_history_length(-1)
static std::string join(char **cmd)
def getVLuminosityBlockRange
void set(const std::string &name, int value)
set the flag, with a run-time name