splitter Namespace Reference

Classes
class	FileObj
Functions
def	appendLinkToList
def	backupOriginal
def	createHtmlPages
def	createMenu
def	extractLinks
def	getFooter
def	getHeader
def	getRunNumberFromFileName
def	main
Variables
dictionary	INDEX = {}
list	LINKS = []
list	PREFIX = sys.argv[3]
list	PROJECT_LOCATION = sys.argv[1]
list	sourceFile = PROJECT_LOCATION+sys.argv[2]

---

Detailed Description

Created on Nov 9, 2010
Updated on Oct 19, 2011

@author: Mantas Stankevicius

---

Function Documentation

def splitter::appendLinkToList	(		line,
			letter
	)

Definition at line 100 of file splitter.py.

                                  :
    if (not INDEX.has_key(letter)):
        subList = [letter, line]
        LINKS.append(subList)
        INDEX[letter] = letter
    else:
        for l in LINKS:
            if l[0] == letter:
                l.append(line)
    

def splitter::backupOriginal

(

)

Definition at line 172 of file splitter.py.

                    :
    fh = open(sourceFile,'r')
    html = fh.read()
    fh.close()

        
    soap = BeautifulSoup(html)
    div = soap.find("div", {"class":"tabs2"})
    # Adding menu of letters at the end of navigation bar
    text = NavigableString(createMenu("All"))
    div.append(text)
#    div.insert(div.__len__(), createMenu("All"))
    
    html = soap.renderContents()
    
    output = open(PROJECT_LOCATION+"/doc/html/"+PREFIX+"All.html", "w")
    output.write(html)
    output.close() 

def splitter::createHtmlPages

(

)

Definition at line 135 of file splitter.py.

                     :
    
    HTMLHeader = getHeader()
    HTMLFooter = getFooter()
    
    for list in LINKS:
        letter = list[0]
        
        html = HTMLHeader
        
        for item in list[1:]:
            html += item+"\n"
        
        html += HTMLFooter
        
        soap = BeautifulSoup(html)
        div = soap.find("div", {"class":"tabs2"})

        text = NavigableString(createMenu(letter))
        div.append(text)

#        div.insert(div.__len__(), createMenu(letter))
        
        html = soap.renderContents()
        
        path = PROJECT_LOCATION+"/doc/html/"+PREFIX+letter+".html"
        output = open(path, "w")
        output.write(html)
        output.close()
        
        if letter == "A":
            output = open(sourceFile, "w")
            output.write(html)
            output.close()  
        
        print PROJECT_LOCATION+"/doc/html/"+PREFIX+letter+".html    Done!"                  

def splitter::createMenu

(

letter

)

Definition at line 111 of file splitter.py.

                      :
    html  = "<div class=\"tabs3\">\n"
    html += "<ul class=\"tablist\">\n"

    letters = []
    for i in INDEX:
        letters.append(i)
    
    letters.sort()
    letters.append("All")
    
    for l in letters:
        c = l
        current = ""
        if c == letter:
            current = " class=\"current\""
            
        html += "<li"+current+"><a href=\""+PREFIX+c+".html\"><span>"+c+"</span></a></li>\n"
    
    html += "</ul>\n"
    html += "</div>\n"
    
    return html

def splitter::extractLinks

(

)

Extracts links from source file 
    from <div class = 'contents'> </div>

Definition at line 67 of file splitter.py.

                  :
    """ Extracts links from source file 
        from <div class = 'contents'> </div>"""
        
    fh = open(sourceFile,'r')
    source = fh.read()
    fh.close()
    
    soup = BeautifulSoup(source)
    div = soup.find("div", {"class":"contents"})
    
    if (div != None):
        content = div.renderContents()
    
    lines = content.split("\n")
    for line in lines:
        if (line.find("<tr>") != -1):
            
            indexFrom = line.rfind(".html\">") + 7
            indexTo = line.rfind("</a>")
            linkText = line[indexFrom:indexTo]
            
            linkTextParts = linkText.split("::")
            
            if len(linkTextParts) == 2:
                tmpLine = line.replace(linkText, linkTextParts[1])
                letter = linkTextParts[1][0].upper()
                appendLinkToList(tmpLine, letter)

            letter = linkText[0].upper()
            appendLinkToList(line, letter)
            

def splitter::getFooter

(

)

Reading source file from end until </table>. 
    After </table> begins list of links (reading from end) 

Definition at line 43 of file splitter.py.

               :
    """ Reading source file from end until </table>. 
        After </table> begins list of links (reading from end) 
    """
    fh = open(sourceFile,'r')
    source = fh.read()
    fh.close()
    
    lines = source.split("\n")
    lines.reverse()
    
    html = []
    enough = False
    
    for line in lines:
        if (not enough):
            html.append(line)
            
        if line.find("</table>") != -1:
            enough = True
        
    html.reverse()            
    return "\n".join(html)   

def splitter::getHeader

(

)

Reading source file until <table>. 
    After <table> begins list of links 

Definition at line 20 of file splitter.py.

               :
    """ Reading source file until <table>. 
        After <table> begins list of links 
    """
    fh = open(sourceFile,'r')
    source = fh.read()
    fh.close()
    
    lines = source.split("\n")

    html = []
    enough = False
    
    for line in lines:
        if line.find("<table>") != -1:
            enough = True
            
        if (not enough):
            html.append(line)
            
    html.append("<table width=\"100%\">")
    return "\n".join(html)

def splitter::getRunNumberFromFileName

(

fileName

)

Definition at line 11 of file splitter.py.

                                      :
    regExp = re.search('(\D+)_(\d+)_(\d+)_(\d+)',fileName)
    if not regExp:
        return -1
    return long(regExp.group(3))
                

def splitter::main

(

)

Definition at line 19 of file splitter.py.

00020           :
00021     if len(sys.argv) < 2:
00022         error = "Usage: splitter fromDir"
00023         exit(error)
00024     sourceDir = sys.argv[1] + '/'
00025 
00026     fileList = ls(sourceDir,".txt")
00027 
00028     fileObjList = {}
00029 
00030     totalSize = 0
00031     for fileName in fileList:
00032         runNumber = getRunNumberFromFileName(fileName)
00033         if runNumber not in fileObjList:
00034             fileObjList[runNumber] = FileObj()
00035             fileObjList[runNumber].run = runNumber 
00036         fileObjList[runNumber].fileNames.append(fileName) 
00037         aCommand  = 'ls -l '+ sourceDir + fileName 
00038         output = commands.getstatusoutput( aCommand )
00039         fileObjList[runNumber].size += int(output[1].split(' ')[4])
00040         totalSize += int(output[1].split(' ')[4]) 
00041 
00042     sortedKeys = fileObjList.keys()
00043     sortedKeys.sort()
00044 
00045     split=13
00046 
00047     dirSize = 0
00048     tmpList = []
00049     for run in sortedKeys:
00050         dirSize += fileObjList[run].size
00051         tmpList.append(fileObjList[run])
00052         if dirSize > totalSize/split or run == sortedKeys[len(sortedKeys)-1]:
00053             newDir = sourceDir + "Run" + str(tmpList[0].run) + "_" + str(tmpList[len(tmpList)-1].run) + "/"
00054             aCommand  = 'mkdir '+ newDir
00055             output = commands.getstatusoutput( aCommand )
00056             print str(100.*dirSize/totalSize) + "% " + "Run" + str(tmpList[0].run) + "_" + str(tmpList[len(tmpList)-1].run) 
00057             for runs in tmpList:
00058                 #print 'cp '+ sourceDir + runs.fileNames[0] + " " + newDir
00059                 cp(sourceDir,newDir,runs.fileNames) 
00060             tmpList = []
00061             dirSize = 0
00062         
00063 
00064 
00065     
00066     print totalSize
00067     print sortedKeys 
00068     exit("ok")    
00069 
00070 
00071 
00072 
00073 
00074 
00075     if not os.path.isdir(destDir):
00076         error = "WARNING: destination directory doesn't exist! Creating it..."
00077         print error
00078         os.mkdir(destDir)
00079     copiedFiles = cp(sourceDir,destDir,fileList)
00080 
00081     if len(copiedFiles) != len(fileList):
00082         error = "ERROR: I couldn't copy all files from castor"
00083         exit(error)
00084 
00085     for fileName in fileList:
00086         fullFileName = destDir + fileName
00087         runNumber = -1;
00088         with open(fullFileName,'r') as file:
00089             for line in file:
00090                 if line.find("Runnumber") != -1:
00091                     tmpRun = int(line.split(' ')[1])
00092                     if runNumber != -1 and tmpRun != runNumber:
00093                         error = "This file (" + fileName + ") contains more than 1 run number! I don't know how to deal with it!"
00094                         exit(error)
00095                     runNumber = int(line.split(' ')[1])
00096         file.close()
00097         newFileName = fileName.replace("None",str(runNumber))
00098         if fileName != newFileName:
00099             aCmd = "mv " + destDir + fileName + " " + destDir + newFileName
00100             print aCmd
00101             output =  commands.getstatusoutput(aCmd)
00102             if output[0] != 0:
00103                 print output[1]
00104         else:
00105             print "WARNING couldn't find keyword None in file " + fileName
00106 
00107 
00108 
00109 
        

---

Variable Documentation

dictionary splitter::INDEX = {}

Definition at line 11 of file splitter.py.

list splitter::LINKS = []

Definition at line 14 of file splitter.py.

Referenced by RPCReadOutMappingWithFastSearch::init(), and RPCReadOutMapping::rawDataFrame().

list splitter::PREFIX = sys.argv[3]

Definition at line 197 of file splitter.py.

list splitter::PROJECT_LOCATION = sys.argv[1]

Definition at line 193 of file splitter.py.

list splitter::sourceFile = PROJECT_LOCATION+sys.argv[2]

Definition at line 195 of file splitter.py.