CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
splitter Namespace Reference

Classes

class  FileObj
 

Functions

def appendLinkToList
 
def backupOriginal
 
def createHtmlPages
 
def createMenu
 
def extractLinks
 
def getFooter
 
def getHeader
 
def getRunNumberFromFileName
 
def main
 

Variables

dictionary INDEX = {}
 
list LINKS = []
 
list PREFIX = sys.argv[3]
 
list PROJECT_LOCATION = sys.argv[1]
 
list sourceFile = PROJECT_LOCATION+sys.argv[2]
 

Detailed Description

Created on Nov 9, 2010
Updated on Oct 19, 2011

@author: Mantas Stankevicius

Function Documentation

def splitter.appendLinkToList (   line,
  letter 
)

Definition at line 100 of file splitter.py.

Referenced by extractLinks().

100 
101 def appendLinkToList(line, letter):
102  if (not INDEX.has_key(letter)):
103  subList = [letter, line]
104  LINKS.append(subList)
105  INDEX[letter] = letter
106  else:
107  for l in LINKS:
108  if l[0] == letter:
109  l.append(line)
110 
splitter.appendLinkToList
def appendLinkToList
Definition: splitter.py:100
def splitter.backupOriginal ( )

Definition at line 172 of file splitter.py.

References createMenu().

172 
173 def backupOriginal():
174  fh = open(sourceFile,'r')
175  html = fh.read()
176  fh.close()
177 
178 
179  soap = BeautifulSoup(html)
180  div = soap.find("div", {"class":"tabs2"})
181  # Adding menu of letters at the end of navigation bar
182  text = NavigableString(createMenu("All"))
183  div.append(text)
184 # div.insert(div.__len__(), createMenu("All"))
185 
186  html = soap.renderContents()
187 
188  output = open(PROJECT_LOCATION+"/doc/html/"+PREFIX+"All.html", "w")
189  output.write(html)
190  output.close()
splitter.backupOriginal
def backupOriginal
Definition: splitter.py:172
splitter.createMenu
def createMenu
Definition: splitter.py:111
def splitter.createHtmlPages ( )

Definition at line 135 of file splitter.py.

References createMenu(), getFooter(), and getHeader().

135 
136 def createHtmlPages():
137 
138  HTMLHeader = getHeader()
139  HTMLFooter = getFooter()
140 
141  for list in LINKS:
142  letter = list[0]
143 
144  html = HTMLHeader
145 
146  for item in list[1:]:
147  html += item+"\n"
148 
149  html += HTMLFooter
150 
151  soap = BeautifulSoup(html)
152  div = soap.find("div", {"class":"tabs2"})
153 
154  text = NavigableString(createMenu(letter))
155  div.append(text)
156 
157 # div.insert(div.__len__(), createMenu(letter))
158 
159  html = soap.renderContents()
160 
161  path = PROJECT_LOCATION+"/doc/html/"+PREFIX+letter+".html"
162  output = open(path, "w")
163  output.write(html)
164  output.close()
165 
166  if letter == "A":
167  output = open(sourceFile, "w")
168  output.write(html)
169  output.close()
170 
171  print PROJECT_LOCATION+"/doc/html/"+PREFIX+letter+".html Done!"
splitter.createHtmlPages
def createHtmlPages
Definition: splitter.py:135
splitter.getHeader
def getHeader
Definition: splitter.py:20
splitter.createMenu
def createMenu
Definition: splitter.py:111
splitter.getFooter
def getFooter
Definition: splitter.py:43
def splitter.createMenu (   letter)

Definition at line 111 of file splitter.py.

Referenced by backupOriginal(), and createHtmlPages().

111 
112 def createMenu(letter):
113  html = "<div class=\"tabs3\">\n"
114  html += "<ul class=\"tablist\">\n"
115 
116  letters = []
117  for i in INDEX:
118  letters.append(i)
119 
120  letters.sort()
121  letters.append("All")
122 
123  for l in letters:
124  c = l
125  current = ""
126  if c == letter:
127  current = " class=\"current\""
128 
129  html += "<li"+current+"><a href=\""+PREFIX+c+".html\"><span>"+c+"</span></a></li>\n"
130 
131  html += "</ul>\n"
132  html += "</div>\n"
133 
134  return html
splitter.createMenu
def createMenu
Definition: splitter.py:111
def splitter.extractLinks ( ) 
Extracts links from source file 
    from <div class = 'contents'> </div>

Definition at line 67 of file splitter.py.

References appendLinkToList(), and pileupCalc.upper.

67 
68 def extractLinks():
69  """ Extracts links from source file
70  from <div class = 'contents'> </div>"""
71 
72  fh = open(sourceFile,'r')
73  source = fh.read()
74  fh.close()
75 
76  soup = BeautifulSoup(source)
77  div = soup.find("div", {"class":"contents"})
78 
79  if (div != None):
80  content = div.renderContents()
81 
82  lines = content.split("\n")
83  for line in lines:
84  if (line.find("<tr>") != -1):
85 
86  indexFrom = line.rfind(".html\">") + 7
87  indexTo = line.rfind("</a>")
88  linkText = line[indexFrom:indexTo]
89 
90  linkTextParts = linkText.split("::")
91 
92  if len(linkTextParts) == 2:
93  tmpLine = line.replace(linkText, linkTextParts[1])
94  letter = linkTextParts[1][0].upper()
95  appendLinkToList(tmpLine, letter)
96 
97  letter = linkText[0].upper()
98  appendLinkToList(line, letter)
99 
splitter.extractLinks
def extractLinks
Definition: splitter.py:67
splitter.appendLinkToList
def appendLinkToList
Definition: splitter.py:100
def splitter.getFooter ( ) 
Reading source file from end until </table>. 
    After </table> begins list of links (reading from end)

Definition at line 43 of file splitter.py.

References join().

Referenced by createHtmlPages().

43 
44 def getFooter():
45  """ Reading source file from end until </table>.
46  After </table> begins list of links (reading from end)
47  """
48  fh = open(sourceFile,'r')
49  source = fh.read()
50  fh.close()
51 
52  lines = source.split("\n")
53  lines.reverse()
54 
55  html = []
56  enough = False
57 
58  for line in lines:
59  if (not enough):
60  html.append(line)
61 
62  if line.find("</table>") != -1:
63  enough = True
64 
65  html.reverse()
66  return "\n".join(html)
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
splitter.getFooter
def getFooter
Definition: splitter.py:43
def splitter.getHeader ( ) 
Reading source file until <table>. 
    After <table> begins list of links

Definition at line 20 of file splitter.py.

References join().

Referenced by createHtmlPages().

20 
21 def getHeader():
22  """ Reading source file until <table>.
23  After <table> begins list of links
24  """
25  fh = open(sourceFile,'r')
26  source = fh.read()
27  fh.close()
28 
29  lines = source.split("\n")
30 
31  html = []
32  enough = False
33 
34  for line in lines:
35  if line.find("<table>") != -1:
36  enough = True
37 
38  if (not enough):
39  html.append(line)
40 
41  html.append("<table width=\"100%\">")
42  return "\n".join(html)
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
splitter.getHeader
def getHeader
Definition: splitter.py:20
def splitter.getRunNumberFromFileName (   fileName)

Definition at line 11 of file splitter.py.

Referenced by main().

11 
12 def getRunNumberFromFileName(fileName):
13  regExp = re.search('(\D+)_(\d+)_(\d+)_(\d+)',fileName)
14  if not regExp:
15  return -1
16  return long(regExp.group(3))
17 
18 
splitter.getRunNumberFromFileName
def getRunNumberFromFileName
Definition: splitter.py:11
def splitter.main ( )

Definition at line 19 of file splitter.py.

References CommonMethods.cp(), cmsRelvalreport.exit, getRunNumberFromFileName(), python.rootplot.utilities.ls(), and split.

19 
20 def main():
21  if len(sys.argv) < 2:
22  error = "Usage: splitter fromDir"
23  exit(error)
24  sourceDir = sys.argv[1] + '/'
25 
26  fileList = ls(sourceDir,".txt")
27 
28  fileObjList = {}
29 
30  totalSize = 0
31  for fileName in fileList:
32  runNumber = getRunNumberFromFileName(fileName)
33  if runNumber not in fileObjList:
34  fileObjList[runNumber] = FileObj()
35  fileObjList[runNumber].run = runNumber
36  fileObjList[runNumber].fileNames.append(fileName)
37  aCommand = 'ls -l '+ sourceDir + fileName
38  output = commands.getstatusoutput( aCommand )
39  fileObjList[runNumber].size += int(output[1].split(' ')[4])
40  totalSize += int(output[1].split(' ')[4])
41 
42  sortedKeys = fileObjList.keys()
43  sortedKeys.sort()
44 
45  split=13
46 
47  dirSize = 0
48  tmpList = []
49  for run in sortedKeys:
50  dirSize += fileObjList[run].size
51  tmpList.append(fileObjList[run])
52  if dirSize > totalSize/split or run == sortedKeys[len(sortedKeys)-1]:
53  newDir = sourceDir + "Run" + str(tmpList[0].run) + "_" + str(tmpList[len(tmpList)-1].run) + "/"
54  aCommand = 'mkdir '+ newDir
55  output = commands.getstatusoutput( aCommand )
56  print str(100.*dirSize/totalSize) + "% " + "Run" + str(tmpList[0].run) + "_" + str(tmpList[len(tmpList)-1].run)
57  for runs in tmpList:
58  #print 'cp '+ sourceDir + runs.fileNames[0] + " " + newDir
59  cp(sourceDir,newDir,runs.fileNames)
60  tmpList = []
61  dirSize = 0
62 
63 
64 
65 
66  print totalSize
67  print sortedKeys
68  exit("ok")
69 
70 
71 
72 
73 
74 
75  if not os.path.isdir(destDir):
76  error = "WARNING: destination directory doesn't exist! Creating it..."
77  print error
78  os.mkdir(destDir)
79  copiedFiles = cp(sourceDir,destDir,fileList)
80 
81  if len(copiedFiles) != len(fileList):
82  error = "ERROR: I couldn't copy all files from castor"
83  exit(error)
84 
85  for fileName in fileList:
86  fullFileName = destDir + fileName
87  runNumber = -1;
88  with open(fullFileName,'r') as file:
89  for line in file:
90  if line.find("Runnumber") != -1:
91  tmpRun = int(line.split(' ')[1])
92  if runNumber != -1 and tmpRun != runNumber:
93  error = "This file (" + fileName + ") contains more than 1 run number! I don't know how to deal with it!"
94  exit(error)
95  runNumber = int(line.split(' ')[1])
96  file.close()
97  newFileName = fileName.replace("None",str(runNumber))
98  if fileName != newFileName:
99  aCmd = "mv " + destDir + fileName + " " + destDir + newFileName
100  print aCmd
101  output = commands.getstatusoutput(aCmd)
102  if output[0] != 0:
103  print output[1]
104  else:
105  print "WARNING couldn't find keyword None in file " + fileName
106 
107 
108 
109 
splitter.main
def main
Definition: splitter.py:19
splitter.getRunNumberFromFileName
def getRunNumberFromFileName
Definition: splitter.py:11
split
double split
Definition: MVATrainer.cc:139

Variable Documentation

dictionary splitter.INDEX = {}

Definition at line 11 of file splitter.py.

list splitter.LINKS = []

Definition at line 14 of file splitter.py.

Referenced by RPCReadOutMappingWithFastSearch.init(), and RPCReadOutMapping.rawDataFrame().

list splitter.PREFIX = sys.argv[3]

Definition at line 197 of file splitter.py.

list splitter.PROJECT_LOCATION = sys.argv[1]

Definition at line 193 of file splitter.py.

list splitter.sourceFile = PROJECT_LOCATION+sys.argv[2]

Definition at line 195 of file splitter.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5