confdbOfflineConverter.OfflineConverter Class Reference

Public Member Functions
def	__init__ (self, version='v3', database='run3', url=None, verbose=False, proxy=False, proxyHost='localhost', proxyPort='8080')
def	query (self, *args)

Static Public Member Functions
def	CheckTempDirectory (dir)

Public Attributes
	baseDir
	baseUrl
	connect
	jars
	javaCmd
	proxy
	proxy_connect_args
	proxyHost
	proxyPort
	verbose
	version
	workDir

Static Public Attributes
	databases
	dbparams
	ips_for_proxy

Detailed Description

Definition at line 13 of file confdbOfflineConverter.py.

Constructor & Destructor Documentation

__init__()

def confdbOfflineConverter.OfflineConverter.__init__	(		self,
			version = 'v3',
			database = 'run3',
			url = None,
			verbose = False,
			proxy = False,
			proxyHost = 'localhost',
			proxyPort = '8080'
	)

Definition at line 72 of file confdbOfflineConverter.py.

    def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
                 proxy = False, proxyHost = 'localhost', proxyPort = '8080'): 
        self.verbose = verbose
        self.version = version
        self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
        self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
        self.jars    = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
        if version=='v2':
            #legacy driver for run2 gui
            self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
        self.workDir = ''
        self.proxy = proxy
        self.proxyHost = proxyHost
        self.proxyPort = proxyPort
 
        # check the schema version
        if version not in self.databases:
            # unsupported database version
            sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)
 
        # check the database
        if database in self.databases[version]:
            # load the connection parameters for the given database
            self.connect = self.databases[version][database]
        else:
            # unsupported database
            sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
            sys.exit(1)
 
        if self.proxy:
            self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
            temp_connect = []
            for entry in self.connect:
                for key,item in self.ips_for_proxy.items():
                    entry = entry.replace(key,item)
                temp_connect.append(entry.replace(key,item))
            self.connect  = tuple(temp_connect)
        else:
            self.proxy_connect_args = ()
 
        # check for a custom base URL
        if url is not None:
            self.baseUrl = url
 
        # try to read the .jar files from AFS, or download them
        if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
            # read the .jar fles from AFS
            self.workDir = self.baseDir
        else:
            # try to use $CMSSW_BASE/tmp
            self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
            if not self.workDir:
                # try to use $TMP
                self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
            if not self.workDir:
                # create a new temporary directory, and install a cleanup callback
                self.workDir = tempfile.mkdtemp()
                atexit.register(shutil.rmtree, self.workDir)
            # download the .jar files
            version_website = requests.get(self.baseUrl+"/../confdb.version").text
            jars_require_update = True
            if os.path.exists(os.path.join(self.workDir,"confdb.version")):
                with open(os.path.join(self.workDir,"confdb.version")) as f:
                    version_existing = f.read()                
                    if version_existing==version_website:
                        jars_require_update = False
 
            if jars_require_update:
                for jar in self.jars:
                    # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
                    handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
                    os.close(handle)
                    request = requests.get(self.baseUrl + '/' + jar)
                    with open(temp,'wb') as f:
                        f.write(request.content)
                    os.rename(temp, self.workDir + '/' + jar)
                #jars updated, write their version
                handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
                os.close(handle)
                with open(temp,'w') as f:
                    f.write(version_website)
                os.rename(temp,os.path.join(self.workDir,"confdb.version"))
 
        # setup the java command line and CLASSPATH
        if self.verbose:
            sys.stderr.write("workDir = %s\n" % self.workDir)
        # use non-blocking random number source /dev/urandom (instead of /dev/random), see:
        #   http://blockdump.blogspot.fr/2012/07/connection-problems-inbound-connection.html
        # deal with timezone region not found
        #   http://stackoverflow.com/questions/9156379/ora-01882-timezone-region-not-found
        # increase the thread stack size from the default of 1 MB to work around java.lang.StackOverflowError errors, see
        #   man java
        self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )
 
 

Member Function Documentation

CheckTempDirectory()

def confdbOfflineConverter.OfflineConverter.CheckTempDirectory (   dir )

static

Definition at line 62 of file confdbOfflineConverter.py.

    def CheckTempDirectory(dir):
        dir = os.path.realpath(dir)
        if not os.path.isdir(dir):
            try:
                os.makedirs(dir)
            except:
                return None
        return dir
 
 

query()

def confdbOfflineConverter.OfflineConverter.query	(		self,
		*	args
	)

Definition at line 167 of file confdbOfflineConverter.py.

    def query(self, *args):
        args = self.javaCmd + self.connect + self.proxy_connect_args + args 
        if self.verbose:
            sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
        sub = subprocess.Popen(
            args,
            stdin = None,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE,
            shell = False,
            universal_newlines = True )
        out,err = sub.communicate()
        # 12_0 uses "HLTSchedule", confdb was changed to make this "schedule", this resets it back for this release
        out = out.replace("fragment.schedule","fragment.HLTSchedule").replace("process.schedule","process.HLTSchedule")
        return out,err
 

References DDTOBRodAlgo.connect, confdbOfflineConverter.OfflineConverter.connect, confdbOfflineConverter.OfflineConverter.javaCmd, join(), confdbOfflineConverter.OfflineConverter.proxy_connect_args, python.rootplot.root2matplotlib.replace(), VisualizationOptions.verbose, L1TdeStage2EMTF.verbose, L1TStage2uGMTMuon.verbose, L1TStage2EMTF.verbose, L1TStage2uGMT.verbose, DQMGenericTnPClient.verbose, dqmTnP::AbstractFitter.verbose, TritonService::FallbackOpts.verbose, L1TStage2OMTF.verbose, DeviationsFromFileSensor2D.verbose, L1TStage2BMTF.verbose, confdbOfflineConverter.OfflineConverter.verbose, L1TStage2MuonComp.verbose, L1TStage2RegionalMuonCandComp.verbose, L1TCaloLayer1RawToDigi.verbose, HIPAlignmentAlgorithm.verbose, BeamSpotDipServer.verbose, o2olib.O2OJobMgr.verbose, L1TStage2uGTCaloLayer2Comp.verbose, OpticalObject.verbose, and L1TdeStage2CaloLayer2.verbose.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

Member Data Documentation

baseDir

confdbOfflineConverter.OfflineConverter.baseDir

Definition at line 75 of file confdbOfflineConverter.py.

baseUrl

confdbOfflineConverter.OfflineConverter.baseUrl

Definition at line 76 of file confdbOfflineConverter.py.

connect

confdbOfflineConverter.OfflineConverter.connect

Definition at line 94 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query(), and o2o_db_cfgmap.DbManagerDAQ.update_hashmap().

databases

confdbOfflineConverter.OfflineConverter.databases

static

Definition at line 31 of file confdbOfflineConverter.py.

dbparams

confdbOfflineConverter.OfflineConverter.dbparams

static

Definition at line 57 of file confdbOfflineConverter.py.

ips_for_proxy

confdbOfflineConverter.OfflineConverter.ips_for_proxy

static

Definition at line 46 of file confdbOfflineConverter.py.

jars

confdbOfflineConverter.OfflineConverter.jars

Definition at line 77 of file confdbOfflineConverter.py.

javaCmd

confdbOfflineConverter.OfflineConverter.javaCmd

Definition at line 163 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query().

proxy

confdbOfflineConverter.OfflineConverter.proxy

Definition at line 82 of file confdbOfflineConverter.py.

proxy_connect_args

confdbOfflineConverter.OfflineConverter.proxy_connect_args

Definition at line 101 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query().

proxyHost

confdbOfflineConverter.OfflineConverter.proxyHost

Definition at line 83 of file confdbOfflineConverter.py.

proxyPort

confdbOfflineConverter.OfflineConverter.proxyPort

Definition at line 84 of file confdbOfflineConverter.py.

verbose

confdbOfflineConverter.OfflineConverter.verbose

Definition at line 73 of file confdbOfflineConverter.py.

Referenced by python.diff_provenance.difference.list_diff(), python.diffProv.difference.list_diff(), python.diff_provenance.difference.module_diff(), python.diffProv.difference.module_diff(), python.diff_provenance.difference.onefilemodules(), python.diffProv.difference.onefilemodules(), core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.process(), core.SkimAnalyzerCount.SkimAnalyzerCount.process(), objects.VertexAnalyzer.VertexAnalyzer.process(), and confdbOfflineConverter.OfflineConverter.query().

version

confdbOfflineConverter.OfflineConverter.version

Definition at line 74 of file confdbOfflineConverter.py.

Referenced by validation.Sample.datasetpattern(), and validation.Sample.filename().

workDir

confdbOfflineConverter.OfflineConverter.workDir

Definition at line 81 of file confdbOfflineConverter.py.