|
|
|
Functions | |
| def | Archive |
| def | CheckPrivateDB |
| def | Cleaner |
| def | ConfirmSize |
| def | CreatePrivateDBForArchival |
| def | DiskUsage |
| def | fileunreg |
| def | GetListFromMasterDB |
| def | Remover |
| def | SendFile |
| def | Transfer |
| def | UpdatePrivateDBForArchival |
Variables | |
| string | adb = "/afs/cern.ch/user/s/smaruyam/CMSSW_2_0_6/src/DQM/Integration/scripts/tmp/archival.db" |
| int | count = 0 |
| string | db = "/afs/cern.ch/user/s/smaruyam/CMSSW_2_0_6/src/DQM/Integration/scripts/tmp/tmp.db" |
| string | dir = "/afs/cern.ch/user/s/smaruyam/CMSSW_2_0_6/src/DQM/Integration/scripts/tmp/" |
| float | disk_threshold = 80.0 |
| tuple | LOGFILE = open('archival_log.txt', 'a') |
| string | targetdir = "/castor/cern.ch/user/s/smaruyam/Archive/" |
| def archive::Archive | ( | ) |
Definition at line 33 of file archive.py.
00033 : 00034 print "Starting Archiver " 00035 if os.path.exists(adb) is False : CreatePrivateDBForArchival() 00036 # count = 0 00037 # DiskUsage() # disabeled for now 00038 Transfer() 00039 # diskusage = False # for test 00040 00041 """ 00042 Disk Usage Check 00043 df out put is assumed as follows. 00044 Filesystem 1K-blocks Used Available Use% Mounted on 00045 /dev/sda3 78012484 2627788 71421864 4% / 00046 /dev/sda1 101086 11487 84380 12% /boot 00047 none 2068904 0 2068904 0% /dev/shm 00048 /dev/sdb1 5046428668 635248096 4154836768 14% /cms/mon/data 00049 cmsnfshome0:/nfshome0 00050 412849344 270910144 120967680 70% /cmsnfshome0/nfshome0 00051 # cmsmon is 5th line from top. 00052 """ def DiskUsage() :
| def archive::CheckPrivateDB | ( | filepath | ) |
Definition at line 140 of file archive.py.
00140 : 00141 string = filepath.split('|') 00142 print "Checking Archival DB for " , string[0] 00143 search = "'" + string[0] + "'" 00144 sqlite = "sqlite3 %s \"select name, size, mtime, atime from t_files where name like %s \"" %(adb, search) 00145 result = os.popen(sqlite) 00146 # print result 00147 check_result = result.readlines() 00148 # print " Read Length is " , len(check_result) 00149 if len(check_result) == 0 : 00150 return False # no entry 00151 else : 00152 return True # entry exist 00153 00154 00155 """ 00156 Tranfer Merged Files 00157 """ def Transfer() :
| def archive::Cleaner | ( | ) |
Definition at line 102 of file archive.py.
00102 : 00103 print "Cleaning File" 00104 flag = True 00105 archivedfiles = GetListFromMasterDB(flag).readlines() 00106 if len(archivedfiles) != 0 : 00107 string = archivedfiles[0].rstrip() 00108 string.split() 00109 filepath = string[0] 00110 Remover(filepath) 00111 if len(archivedfiles) == 0 : print "Cleaning Failed because DB gave a Null List!" 00112 00113 00114 """ 00115 Master DataBase Reference 00116 """ def GetListFromMasterDB(flag) :
| def archive::ConfirmSize | ( | filename, | ||
| size | ||||
| ) |
Definition at line 76 of file archive.py.
00076 : 00077 print "Confirming Copied File Size for " , filename 00078 time.sleep(10) 00079 # ls = "ls -l " + pnfsdir + filename 00080 ls = "rfdir " + targetdir + filename 00081 # print ls 00082 result = os.popen('%s' %ls) 00083 # print result 00084 for line in result : 00085 error_check = "%s%s: No such file or directory" %(targetdir, filename) 00086 if line == error_check : 00087 return False 00088 else : 00089 # print line 00090 string = line.split() 00091 if len(string) == 9 : 00092 # print string[4] 00093 if string[4] == size : 00094 print "Size matched, Copy Success" 00095 return True 00096 else : return False 00097 else : return False 00098 00099 """ 00100 File Cleaner 00101 """ def Cleaner() :
| def archive::CreatePrivateDBForArchival | ( | ) |
Definition at line 194 of file archive.py.
00194 : 00195 print "Creating Archival DB" 00196 sqlite = "sqlite3 %s \"CREATE TABLE t_files (id integer primary key autoincrement, name text unique not null, size integer not null, mtime integer not null, atime text not null); \" " %adb 00197 result = os.popen(sqlite) 00198 check_result = result.readlines() 00199 # print " Read Length is " , len(check_result) 00200 if len(check_result) == 0 : 00201 return True 00202 else : 00203 print result 00204 return False 00205 00206 00207 """ 00208 copy file to dcache or castor 00209 """ def SendFile(filepath) :
| def archive::DiskUsage | ( | ) |
Definition at line 53 of file archive.py.
00053 : 00054 print "Checking Disk Usage" 00055 df_file=os.popen('df') 00056 usage = False 00057 lines = df_file.readlines() 00058 list = lines[4].split() # 5th line from top. Split at tab or white space 00059 string = list[4][:-1] # NEED check for cmsmon # 00060 fusage = float(string) 00061 print fusage 00062 if fusage > disk_threshold : # disk is more than 80% full 00063 print "Disk Usage too high" 00064 usage = True 00065 if usage == True : 00066 Cleaner() 00067 else : 00068 print "Disk Usage is low enough" 00069 00070 00071 """ 00072 check copied file and its size 00073 rfdir out put is assumed as follows. 00074 -rw-r--r-- 1 smaruyam zh 10 May 08 22:35 /castor/cern.ch/user/s/smaruyam/Archive/test.txt 00075 """ def ConfirmSize(filename, size) :
| def archive::fileunreg | ( | db, | ||
| file, | ||||
| logfile | ||||
| ) |
Definition at line 18 of file archive.py.
00018 : 00019 tmpdb = '/cms/mon/data/.dropbox_test/dqm-tmp.db' 00020 newdb = db[:-3]+'-new.db' 00021 server = 'srv-c2d05-19' 00022 if os.path.exists(tmpdb): os.remove(tmpdb) 00023 logfile.write(os.popen('scp '+server+ ':'+db+' '+tmpdb).read()) 00024 logfile.write('*** File UnRegister ***\n') 00025 logfile.write(os.popen('visDQMUnregisterFile '+ tmpdb +' ' + file).read()) 00026 logfile.write(os.popen('scp '+tmpdb+' '+server+':'+newdb).read()) 00027 os.remove(tmpdb) 00028 logfile.write(os.popen('ssh '+server+' -t mv '+newdb+' '+db).read()) 00029 00030 """ 00031 Archiving Files 00032 """ def Archive() :
| def archive::GetListFromMasterDB | ( | flag | ) |
Definition at line 117 of file archive.py.
00117 : 00118 print "Getting Merged File List from Master DB" 00119 if flag is False : # Retrive Merged File Info Only 00120 string = "'" + dir + "DQM_V%_R%.root" + "'" 00121 # print string 00122 search = "'%RPC%'" 00123 sqlite = "sqlite3 %s \"select name, size, mtime from t_files where name like %s and not name like %s \" " %(db, string, search) 00124 result = os.popen(sqlite) 00125 # print result 00126 return result 00127 if flag is True : # Retrive Merged and Un-Merged File Info # Need check on Un-Merged Files DB (where is it?) 00128 sqlite = "sqlite3 %s \"select name, mtime from t_files where name like '%DQM%.root' order by mtime asc\"" %db 00129 result = os.popen(sqlite) 00130 # print result 00131 return result 00132 00133 00134 """ 00135 Private DataBase Check 00136 Private DB is asuumed as follows. 00137 CREATE TABLE t_files (id integer primary key autoincrement, name text unique 00138 not null, size integer not null, mtime integer not null, atime text not null); 00139 """ def CheckPrivateDB(filepath) :
| def archive::Remover | ( | filepath | ) |
Definition at line 258 of file archive.py.
00258 : 00259 print "Removeing File: %s" %filepath 00260 result = os.popen('rm -f %s' %filepath) 00261 for line in result.readlines() : 00262 if line == "" : ## now error check rm 00263 print "File Removed" 00264 fileunreg(db, filepath, LOGFILE) #def fileunreg(db,file,logfile): 00265 else : 00266 print "Cannot Remove This File because ", line 00267 00268 """ 00269 Main Script 00270 """ if __name__ == "__main__":
| def archive::SendFile | ( | filepath | ) |
Definition at line 210 of file archive.py.
00210 : 00211 print "Shipping %s to %s" %(filepath, targetdir) 00212 # Assumed root name = DQM_Vxx_Ryyyyyyyyy-Rzzzzzzzzz.root 00213 filename = filepath[-34:] # file name only 00214 # print filename 00215 00216 # cdir = os.popen('pwd') 00217 # pwd = cdir.readline().rstrip() 00218 # print pwd 00219 # print "srmcp -debug==true file:////%s/%s \"%s%s\"" %(pwd, filepath, targetdir, filename) 00220 00221 # result = os.popen('srmcp -debug==true file:////%s/%s "%s%s"' %(pwd, filepath, targetdir, filename) ) 00222 result = os.popen('rfcp %s %s' %(filepath, targetdir) ) 00223 # print result 00224 00225 # ls = "ls -l " + filename 00226 ls = "rfdir %s%s" %(targetdir, filename) 00227 # print ls 00228 time.sleep(10) 00229 result = os.popen('%s' %ls) 00230 # print result 00231 size = "" 00232 for line in result.readlines() : 00233 # print line 00234 string = line.split() 00235 # print string 00236 if len(string) == 9 : 00237 # print string[4] 00238 size = string[4] 00239 success = False 00240 counter = 0 00241 while success == False and counter < 5 : # repeat up to five times 00242 counter += 1 00243 success = ConfirmSize(filename, size) 00244 if success is False : ## now error check srmcp 00245 print "File Transfer Failed. This is %d time(s) Failure: " %counter 00246 else : continue 00247 if success is True : 00248 print "%s is copyied " %filepath 00249 update = UpdatePrivateDBForArchival(filepath) 00250 if update is False : print "Updating Private DB Failed !" 00251 time.sleep(10) 00252 if success is False : print "Copying File was Failed! " 00253 00254 00255 """ 00256 Remove file from disk 00257 """ def Remover(filepath) :
| def archive::Transfer | ( | ) |
Definition at line 158 of file archive.py.
00158 : 00159 print "Starting File Transfer" 00160 flag = False 00161 mergedfiles = GetListFromMasterDB(flag) # Retrieve Merged File List 00162 archived = True 00163 for file in mergedfiles : 00164 # print file 00165 string = file.split('|') 00166 archived = CheckPrivateDB(string[0]) 00167 if archived is False : 00168 # print "Here is Root File List ", mergedfiles 00169 SendFile(string[0]) 00170 else : continue 00171 00172 00173 """ 00174 Update Private DB for Archival Information 00175 """ def UpdatePrivateDBForArchival(filepath) :
| def archive::UpdatePrivateDBForArchival | ( | filepath | ) |
Definition at line 176 of file archive.py.
00176 : 00177 print "Updating Archival DB for " , filepath 00178 name = filepath 00179 size = os.path.getsize(filepath) 00180 mtime = os.path.getmtime(filepath) 00181 tm = time.localtime(time.time()) 00182 atime = time.strftime("%Y/%m/%d", tm) 00183 sqlite = "sqlite3 %s \"insert into t_files(name, size, mtime, atime) values('%s', '%d', '%d', '%s')\"" %(adb, name, size, mtime, atime) 00184 result = os.popen(sqlite) 00185 check_result = result.readlines() 00186 # print " Read Length is " , len(check_result) 00187 if len(check_result) == 0 : 00188 return True 00189 else : return False 00190 00191 """ 00192 Create a DB if not present 00193 """ def CreatePrivateDBForArchival() :
string archive::adb = "/afs/cern.ch/user/s/smaruyam/CMSSW_2_0_6/src/DQM/Integration/scripts/tmp/archival.db" [static] |
Definition at line 10 of file archive.py.
int archive::count = 0 [static] |
Definition at line 274 of file archive.py.
string archive::db = "/afs/cern.ch/user/s/smaruyam/CMSSW_2_0_6/src/DQM/Integration/scripts/tmp/tmp.db" [static] |
Definition at line 9 of file archive.py.
Referenced by SiStripCommissioningOfflineDbClient::createHistos(), IgApplication::dumpDatabase(), fillRecordToTypeMap(), DBlmapReader::GetLMAP(), IgWebServiceRegistry::IgWebServiceRegistry(), initDb(), edm::RootAutoLibraryLoader::loadAll(), edm::service::LoadAllDictionaries::LoadAllDictionaries(), main(), EcalUncalibRecHitFixedAlphaBetaAlgo< C >::PerformAnalyticFit(), publishObjects(), IgPluginFactoryBase::rebuild(), edm::registerTypes(), and IgQtAppDebugPluginsInfo::update().
string archive::dir = "/afs/cern.ch/user/s/smaruyam/CMSSW_2_0_6/src/DQM/Integration/scripts/tmp/" [static] |
Definition at line 8 of file archive.py.
float archive::disk_threshold = 80.0 [static] |
Definition at line 13 of file archive.py.
tuple archive::LOGFILE = open('archival_log.txt', 'a') [static] |
Definition at line 5 of file archive.py.
string archive::targetdir = "/castor/cern.ch/user/s/smaruyam/Archive/" [static] |
Definition at line 11 of file archive.py.
1.5.4