Mpslibclass.py

Go to the documentation of this file.

# This Jobdatabas-Class interacts with the mps.db file.
# It's member-variables are often called in the mps_... scripts.
#
# Meaning of the database variables: (still need to work on these)
#
# (1) Header
#       header          - version information
#       batchScript     - base script for serial job
#       cfgTemplate     - template for cfg file
#       infiList        - list of input files to be serialized
#       classInf        - batch class information (might contain two ':'-separated)
#       addFiles        - job name for submission
#       driver          - specifies whether merge job is foreseen
#       nJobs           - number of serial jobs (not including merge job)
#       mergeScript     - base script for merge job
#       mssDir          - directory for mass storage (e.g. Castor)
#       updateTime      - time of last update (seconds since 1970)
#       updateTimeHuman - time of last update (human readable)
#       elapsedTime     - seconds since last update
#       mssDirPool      - pool for $mssDir (e.g. cmscaf/cmscafuser)
#       pedeMem         - Memory allocated for pede
#       spare1
#       spare2
#       spare3

# (2) Job-level variables/lists
#       JOBNUMBER   - ADDED, selfexplanatory
#       JOBDIR      - name of job directory (not full path)
#       JOBSTATUS   - status of job
#       JOBRUNTIME  - present CPU time of job
#       JOBNEVT     - number of events processed by job
#       JOBHOST     - presently used to store remark
#       JOBINCR     - CPU increment since last check
#       JOBREMARK   - comment
#       JOBSP1      - spare
#       JOBSP2      - possible weight for pede
#       JOBSP3      - possible name as given to mps_setup.pl -N <name> ...
#       JOBID       - ID of the LSF/HTCondor job

import datetime
import time
import os
import sys

#-------------------------------------------------------------------------------
class jobdatabase:

    JOBNUMBER, JOBDIR, JOBID, JOBSTATUS, JOBNTRY, JOBRUNTIME, JOBNEVT, JOBHOST, JOBINCR, \
    JOBREMARK, JOBSP1, JOBSP2, JOBSP3 = ([] for i in range(13))

    header, batchScript, cfgTemplate, infiList, classInf, addFiles, driver, mergeScript, \
    mssDir, updateTimeHuman, mssDirPool, spare1, spare2, spare3 = ('' for i in range(14))

    updateTime, elapsedTime, pedeMem , nJobs = -1, -1, -1, -1

    #-------------------------------------------------------------------------------
    # parses the mps.db file into the member variables and arrays
    __default_db_file_name = "mps.db"
    def read_db(self, db_file_name = __default_db_file_name):
        try:
            DBFILE = open(db_file_name,'r')
        except IOError as e:
            if e.args != (2, 'No such file or directory'):
                raise
            else:
                if db_file_name == jobdatabase.__default_db_file_name:
                    msg = ("No 'mps.db' found. Make sure you are in a campaign "
                           "directory and that the campaign is set up.")
                else:
                    msg = "Database file '"+db_file_name+"' not found. Exiting."
                print msg
                sys.exit(1)

        #read infolines at the top, used rstrip to delete the '\n'
        self.header          = DBFILE.readline().strip()
        self.batchScript     = DBFILE.readline().rstrip('\n')
        self.cfgTemplate     = DBFILE.readline().rstrip('\n')
        self.infiList        = DBFILE.readline().rstrip('\n')
        self.classInf        = DBFILE.readline().rstrip('\n')   #formerly named 'class' ->conflict
        self.addFiles        = DBFILE.readline().rstrip('\n')
        self.driver          = DBFILE.readline().rstrip('\n')
        self.mergeScript     = DBFILE.readline().rstrip('\n')
        self.mssDir          = DBFILE.readline().rstrip('\n')
        self.updateTime      = int(DBFILE.readline())
        self.updateTimeHuman = DBFILE.readline().rstrip('\n')
        self.elapsedTime     = int(DBFILE.readline())
        self.mssDirPool      = DBFILE.readline().rstrip('\n')
        self.pedeMem         = int(DBFILE.readline())
        self.spare1          = DBFILE.readline().rstrip('\n')
        self.spare2          = DBFILE.readline().rstrip('\n')
        self.spare3          = DBFILE.readline().rstrip('\n')

        #read actual jobinfo into arrays
        self.nJobs = 0
        milleJobs = 0


        for line in DBFILE:
            if line.strip() == "": continue # ignore empty lines
            line = line.rstrip('\n')        # removes the pesky \n from line
            parts = line.split(":")         # read each line and split into parts list
            self.JOBNUMBER.append(int(parts[0]))
            self.JOBDIR.append(parts[1].strip())
            self.JOBID.append(parts[2])
            self.JOBSTATUS.append(parts[3].strip())
            self.JOBNTRY.append(int(parts[4]))
            self.JOBRUNTIME.append(int(parts[5]))   #int float?
            self.JOBNEVT.append(int(parts[6]))
            self.JOBHOST.append(parts[7].strip())
            self.JOBINCR.append(int(parts[8]))
            self.JOBREMARK.append(parts[9].strip())
            self.JOBSP1.append(parts[10].strip())
            self.JOBSP2.append(parts[11].strip())
            self.JOBSP3.append(parts[12].strip())

            #count number of jobs
            if not self.JOBDIR[self.nJobs].startswith("jobm"):
                milleJobs += 1
            self.nJobs += 1
        self.nJobs = milleJobs

        DBFILE.close()



    #-------------------------------------------------------------------------------
    # prints the member varaiables and arrays to the terminal
    def print_memdb(self):
        #print metainfo
        print "\n=== mps database printout ===\n"
        print self.header
        print 'Script:\t\t',    self.batchScript
        print 'cfg:\t\t',       self.cfgTemplate
        print 'files:\t\t',     self.infiList
        print 'class:\t\t',     self.classInf
        print 'name:\t\t',      self.addFiles
        print 'driver:\t\t',    self.driver
        print 'mergeScript:\t', self.mergeScript
        print 'mssDir:\t\t',    self.mssDir
        print 'updateTime:\t',  self.updateTimeHuman
        print 'elapsed:\t',     self.elapsedTime
        print 'mssDirPool:\t',  self.mssDirPool
        print 'pedeMem:\t',             self.pedeMem, '\n'

        #print interesting Job-level lists ---- to add: t/evt, fix remarks
        print '###     dir      jobid    stat  try  rtime      nevt  remark   weight  name'
        print "------------------------------------------------------------------------------"
        for i in xrange(self.nJobs):
            print '%03d  %6s  %9s  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
                self.JOBNUMBER[i],
                self.JOBDIR[i],
                self.JOBID[i],
                self.JOBSTATUS[i],
                self.JOBNTRY[i],
                self.JOBRUNTIME[i],
                self.JOBNEVT[i],
                self.JOBHOST[i],
                self.JOBSP2[i],
                self.JOBSP3[i])

        #print merge Jobs if merge mode
        if self.driver == 'merge':
            for i in xrange(self.nJobs,len(self.JOBDIR)):
                print '%s  %6s  %9s  %6s  %3d  %5d  %8d  %8s  %5s  %s' % (
                    'MMM',
                    self.JOBDIR[i],
                    self.JOBID[i],
                    self.JOBSTATUS[i],
                    self.JOBNTRY[i],
                    self.JOBRUNTIME[i],
                    self.JOBNEVT[i],
                    self.JOBHOST[i],
                    self.JOBSP2[i],
                    self.JOBSP3[i])

        #print summed info
        totalEvents = sum(self.JOBNEVT[:self.nJobs])
        totalCpu    = sum(self.JOBRUNTIME[:self.nJobs])
        meanCpuPerEvent = 0.
        if totalEvents > 0:
            meanCpuPerEvent = float(totalCpu)/totalEvents
        print "------------------------------------------------------------------------------"
        print "\t\t\t\t\tEvent total:\t",       totalEvents
        print "\t\t\t\t\tCPU total:\t",         totalCpu,               's'
        print "\t\t\t\t\tMean CPU/event:\t",meanCpuPerEvent,'s'





    #-------------------------------------------------------------------------------
    # writes a new mps.db file from the members. Replaces the old mps.db
    def write_db(self):
        self.header = "mps database schema 3.2"
        self.currentTime = int(time.time())
        self.elapsedTime = 0;
        if self.updateTime != 0:
            self.elapsedTime = self.currentTime - self.updateTime
        self.updateTime = self.currentTime
        self.updateTimeHuman = str(datetime.datetime.today())   #no timezone :(
        self.spare1 = "-- unused --"
        self.spare2 = "-- unused --"
        self.spare3 = "-- unused --"

        #if mps.db already exists, backup as mps.db~ (in case of interupt during write)
        os.system('[[ -a mps.db ]] && cp -p mps.db mps.db~')

        #write mps.db header
        DBFILE = open ("mps.db", "w")
        headData = [ self.header, self.batchScript, self.cfgTemplate, self.infiList,
                     self.classInf, self.addFiles, self.driver, self.mergeScript,
                     self.mssDir, self.updateTime, self.updateTimeHuman,
                     self.elapsedTime, self.mssDirPool, self.pedeMem,
                     self.spare1, self.spare2, self.spare3 ]
        for item in headData:
            DBFILE.write("%s\n" % item)

        #write mps.db jobinfo
        for i in xrange(len(self.JOBID)):
            DBFILE.write('%03d:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s:%s\n' %
                         (i+1,
                          self.JOBDIR[i],
                          self.JOBID[i],
                          self.JOBSTATUS[i],
                          self.JOBNTRY[i],
                          self.JOBRUNTIME[i],
                          self.JOBNEVT[i],
                          self.JOBHOST[i],
                          self.JOBINCR[i],
                          self.JOBREMARK[i],
                          self.JOBSP1[i],
                          self.JOBSP2[i],
                          self.JOBSP3[i]))
        DBFILE.close()

    #-------------------------------------------------------------------------------
    # returns job class as stored in db
    # one and only argument may be "mille" or "pede" for mille or pede jobs
    def get_class(self, argument=''):
        CLASSES = self.classInf.split(':')
        if len(CLASSES)<1 or len(CLASSES)>2:
            print '\nget_class():\n  class must be of the form \'class\' or \'classMille:classPede\', but is \'%s\'!\n\n', classInf
            sys.exit(1)
        elif argument == 'mille':
            return CLASSES[0]
        elif argument == 'pede':
            if len(CLASSES) == 1:
                return CLASSES[0]
            elif len(CLASSES) == 2:
                return CLASSES[1]
        else:
            print '\nget_class():\n  Know class only for \'mille\' or \'pede\', not %s!\n\n' %argument
            sys.exit(1)