CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
splitter Namespace Reference

Classes

class  FileObj
 

Functions

def backupOriginal
 
def createHtmlPages
 
def createMenu
 
def extractLinks
 
def getFooter
 
def getHeader
 
def getRunNumberFromFileName
 
def main
 

Variables

dictionary INDEX = {}
 
list LINKS = []
 
list PREFIX = sys.argv[3]
 
list PROJECT_LOCATION = sys.argv[1]
 
list sourceFile = PROJECT_LOCATION+sys.argv[2]
 

Detailed Description

Created on Nov 9, 2010

@author: Mantas Stankevicius

Function Documentation

def splitter.backupOriginal ( )

Definition at line 157 of file splitter.py.

References createMenu().

157 
158 def backupOriginal():
159  fh = open(sourceFile,'r')
160  html = fh.read()
161  fh.close()
162 
163 
164  soap = BeautifulSoup(html)
165  div = soap.find("div", {"class":"tabs2"})
166  # Adding menu of letters at the end of navigation bar
167  text = NavigableString(createMenu("All"))
168  div.append(text)
169 # div.insert(div.__len__(), createMenu("All"))
170 
171  html = soap.renderContents()
172 
173  output = open(PROJECT_LOCATION+"/doc/html/"+PREFIX+"All.html", "w")
174  output.write(html)
175  output.close()
splitter.backupOriginal
def backupOriginal
Definition: splitter.py:157
splitter.createMenu
def createMenu
Definition: splitter.py:96
def splitter.createHtmlPages ( )

Definition at line 120 of file splitter.py.

References createMenu(), getFooter(), and getHeader().

120 
121 def createHtmlPages():
122 
123  HTMLHeader = getHeader()
124  HTMLFooter = getFooter()
125 
126  for list in LINKS:
127  letter = list[0]
128 
129  html = HTMLHeader
130 
131  for item in list[1:]:
132  html += item+"\n"
133 
134  html += HTMLFooter
135 
136  soap = BeautifulSoup(html)
137  div = soap.find("div", {"class":"tabs2"})
138 
139  text = NavigableString(createMenu(letter))
140  div.append(text)
141 
142 # div.insert(div.__len__(), createMenu(letter))
143 
144  html = soap.renderContents()
145 
146  path = PROJECT_LOCATION+"/doc/html/"+PREFIX+letter+".html"
147  output = open(path, "w")
148  output.write(html)
149  output.close()
150 
151  if letter == "A":
152  output = open(sourceFile, "w")
153  output.write(html)
154  output.close()
155 
156  print PROJECT_LOCATION+"/doc/html/"+PREFIX+letter+".html Done!"
splitter.createHtmlPages
def createHtmlPages
Definition: splitter.py:120
splitter.getHeader
def getHeader
Definition: splitter.py:19
splitter.createMenu
def createMenu
Definition: splitter.py:96
splitter.getFooter
def getFooter
Definition: splitter.py:42
def splitter.createMenu (   letter)

Definition at line 96 of file splitter.py.

Referenced by backupOriginal(), and createHtmlPages().

96 
97 def createMenu(letter):
98  html = "<div class=\"tabs3\">\n"
99  html += "<ul class=\"tablist\">\n"
100 
101  letters = []
102  for i in INDEX:
103  letters.append(i)
104 
105  letters.sort()
106  letters.append("All")
107 
108  for l in letters:
109  c = l
110  current = ""
111  if c == letter:
112  current = " class=\"current\""
113 
114  html += "<li"+current+"><a href=\""+PREFIX+c+".html\"><span>"+c+"</span></a></li>\n"
115 
116  html += "</ul>\n"
117  html += "</div>\n"
118 
119  return html
splitter.createMenu
def createMenu
Definition: splitter.py:96
def splitter.extractLinks ( ) 
Extracts links from source file 
    from <div class = 'contents'> </div>

Definition at line 66 of file splitter.py.

66 
67 def extractLinks():
68  """ Extracts links from source file
69  from <div class = 'contents'> </div>"""
70 
71  fh = open(sourceFile,'r')
72  source = fh.read()
73  fh.close()
74 
75  soup = BeautifulSoup(source)
76  div = soup.find("div", {"class":"contents"})
77 
78  if (div != None):
79  content = div.renderContents()
80 
81  lines = content.split("\n")
82  for line in lines:
83  if (line.find("<tr>") != -1):
84  index = line.rfind(".html\">")
85  # First letter of link
86  letter = line[index+7]
87  letter = letter.upper()
88  if (not INDEX.has_key(letter)):
89  subList = [letter, line]
90  LINKS.append(subList)
91  INDEX[letter] = letter
92  else:
93  for l in LINKS:
94  if l[0] == letter:
95  l.append(line)
splitter.extractLinks
def extractLinks
Definition: splitter.py:66
def splitter.getFooter ( ) 
Reading source file from end until </table>. 
    After </table> begins list of links (reading from end)

Definition at line 42 of file splitter.py.

References join().

Referenced by createHtmlPages().

42 
43 def getFooter():
44  """ Reading source file from end until </table>.
45  After </table> begins list of links (reading from end)
46  """
47  fh = open(sourceFile,'r')
48  source = fh.read()
49  fh.close()
50 
51  lines = source.split("\n")
52  lines.reverse()
53 
54  html = []
55  enough = False
56 
57  for line in lines:
58  if (not enough):
59  html.append(line)
60 
61  if line.find("</table>") != -1:
62  enough = True
63 
64  html.reverse()
65  return "\n".join(html)
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
splitter.getFooter
def getFooter
Definition: splitter.py:42
def splitter.getHeader ( ) 
Reading source file until <table>. 
    After <table> begins list of links

Definition at line 19 of file splitter.py.

References join().

Referenced by createHtmlPages().

19 
20 def getHeader():
21  """ Reading source file until <table>.
22  After <table> begins list of links
23  """
24  fh = open(sourceFile,'r')
25  source = fh.read()
26  fh.close()
27 
28  lines = source.split("\n")
29 
30  html = []
31  enough = False
32 
33  for line in lines:
34  if line.find("<table>") != -1:
35  enough = True
36 
37  if (not enough):
38  html.append(line)
39 
40  html.append("<table width=\"100%\">")
41  return "\n".join(html)
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
splitter.getHeader
def getHeader
Definition: splitter.py:19
def splitter.getRunNumberFromFileName (   fileName)

Definition at line 11 of file splitter.py.

Referenced by main().

11 
12 def getRunNumberFromFileName(fileName):
13  regExp = re.search('(\D+)_(\d+)_(\d+)_(\d+)',fileName)
14  if not regExp:
15  return -1
16  return long(regExp.group(3))
17 
18 
splitter.getRunNumberFromFileName
def getRunNumberFromFileName
Definition: splitter.py:11
def splitter.main ( )

Definition at line 19 of file splitter.py.

References CommonMethods.cp(), cmsRelvalreport.exit, getRunNumberFromFileName(), python.rootplot.utilities.ls(), and split.

19 
20 def main():
21  if len(sys.argv) < 2:
22  error = "Usage: splitter fromDir"
23  exit(error)
24  sourceDir = sys.argv[1] + '/'
25 
26  fileList = ls(sourceDir,".txt")
27 
28  fileObjList = {}
29 
30  totalSize = 0
31  for fileName in fileList:
32  runNumber = getRunNumberFromFileName(fileName)
33  if runNumber not in fileObjList:
34  fileObjList[runNumber] = FileObj()
35  fileObjList[runNumber].run = runNumber
36  fileObjList[runNumber].fileNames.append(fileName)
37  aCommand = 'ls -l '+ sourceDir + fileName
38  output = commands.getstatusoutput( aCommand )
39  fileObjList[runNumber].size += int(output[1].split(' ')[4])
40  totalSize += int(output[1].split(' ')[4])
41 
42  sortedKeys = fileObjList.keys()
43  sortedKeys.sort()
44 
45  split=13
46 
47  dirSize = 0
48  tmpList = []
49  for run in sortedKeys:
50  dirSize += fileObjList[run].size
51  tmpList.append(fileObjList[run])
52  if dirSize > totalSize/split or run == sortedKeys[len(sortedKeys)-1]:
53  newDir = sourceDir + "Run" + str(tmpList[0].run) + "_" + str(tmpList[len(tmpList)-1].run) + "/"
54  aCommand = 'mkdir '+ newDir
55  output = commands.getstatusoutput( aCommand )
56  print str(100.*dirSize/totalSize) + "% " + "Run" + str(tmpList[0].run) + "_" + str(tmpList[len(tmpList)-1].run)
57  for runs in tmpList:
58  #print 'cp '+ sourceDir + runs.fileNames[0] + " " + newDir
59  cp(sourceDir,newDir,runs.fileNames)
60  tmpList = []
61  dirSize = 0
62 
63 
64 
65 
66  print totalSize
67  print sortedKeys
68  exit("ok")
69 
70 
71 
72 
73 
74 
75  if not os.path.isdir(destDir):
76  error = "WARNING: destination directory doesn't exist! Creating it..."
77  print error
78  os.mkdir(destDir)
79  copiedFiles = cp(sourceDir,destDir,fileList)
80 
81  if len(copiedFiles) != len(fileList):
82  error = "ERROR: I couldn't copy all files from castor"
83  exit(error)
84 
85  for fileName in fileList:
86  fullFileName = destDir + fileName
87  runNumber = -1;
88  with open(fullFileName,'r') as file:
89  for line in file:
90  if line.find("Runnumber") != -1:
91  tmpRun = int(line.split(' ')[1])
92  if runNumber != -1 and tmpRun != runNumber:
93  error = "This file (" + fileName + ") contains more than 1 run number! I don't know how to deal with it!"
94  exit(error)
95  runNumber = int(line.split(' ')[1])
96  file.close()
97  newFileName = fileName.replace("None",str(runNumber))
98  if fileName != newFileName:
99  aCmd = "mv " + destDir + fileName + " " + destDir + newFileName
100  print aCmd
101  output = commands.getstatusoutput(aCmd)
102  if output[0] != 0:
103  print output[1]
104  else:
105  print "WARNING couldn't find keyword None in file " + fileName
106 
107 
108 
109 
splitter.main
def main
Definition: splitter.py:19
splitter.getRunNumberFromFileName
def getRunNumberFromFileName
Definition: splitter.py:11
split
double split
Definition: MVATrainer.cc:139

Variable Documentation

dictionary splitter.INDEX = {}

Definition at line 10 of file splitter.py.

list splitter.LINKS = []

Definition at line 13 of file splitter.py.

Referenced by RPCReadOutMappingWithFastSearch.init(), and RPCReadOutMapping.rawDataFrame().

list splitter.PREFIX = sys.argv[3]

Definition at line 182 of file splitter.py.

list splitter.PROJECT_LOCATION = sys.argv[1]

Definition at line 178 of file splitter.py.

list splitter.sourceFile = PROJECT_LOCATION+sys.argv[2]

Definition at line 180 of file splitter.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5