Functions
def	checkFile ()

def	copy (args)

def	run (args)

def	saveFileForImportErrors (datef, withMetadata=False)

def	upload (args)

Function Documentation

def popcon2dropbox.checkFile ( )

Definition at line 35 of file popcon2dropbox.py.

References harvestTrackValidationPlots.str.

Referenced by run().

 def checkFile():
     # check if the expected input file is there... 
     # exit code < 0 => error
     # exit code = 0 => skip
     # exit code = 1 => import                                                                                                                             
     if not os.path.exists( dbFileName ):
        logger.error('The file expected as an input %s has not been found.'%dbFileName )
        return -1
 
     empty = True
     try:
        dbcon = sqlite3.connect( dbFileName )
        dbcur = dbcon.cursor()
        dbcur.execute('SELECT * FROM IOV')
        rows = dbcur.fetchall()
        for r in rows:
           empty = False
        dbcon.close()
        if empty:
            logger.warning('The file expected as an input %s contains no data. The import will be skipped.'%dbFileName )
            return 0
        return 1
     except Exception as e:
        logger.error('Check on input data failed: %s' %str(e))
        return -2
 

def popcon2dropbox.copy ( args )

Definition at line 123 of file popcon2dropbox.py.

References saveFileForImportErrors(), and harvestTrackValidationPlots.str.

Referenced by DigiCollectionFP420.add(), AlignmentSurfaceDeformations.add(), GeometricDet.addComponents(), MatacqProducer.addMatacqData(), AlignableExtras.alignmentErrors(), AlignableDet.alignmentErrors(), AlignableCSCEndcap.alignmentErrors(), AlignableDTBarrel.alignmentErrors(), AlignableComposite.alignmentErrors(), AlignableMuon.alignmentErrors(), AlignableTracker.alignmentErrors(), AlignableExtras.alignments(), AlignableDet.alignments(), AlignableCSCEndcap.alignments(), AlignableDTBarrel.alignments(), AlignableComposite.alignments(), AlignableMuon.alignments(), AlignableTracker.alignments(), LHEWriter.analyze(), DTResidualCalibration.analyze(), DTEfficiencyTask.analyze(), DTResolutionAnalysisTask.analyze(), FFTJetImageRecorder.analyze(), FFTJetPileupAnalyzer.analyze(), GeometryAligner.applyAlignments(), SegmentsTrackAssociator.associate(), GeometryAligner.attachSurfaceDeformations(), EmbeddingLHEProducer.beginRunProduce(), ecaldqm::MESetNonObject.book(), ecaldqm::MESetEcal.book(), L1ExtraDQM::L1ExtraMonElement< CollectionType >.bookhistograms(), MuonTransientTrackingRecHitBreaker.breakInSubRecHits(), CSCGeometryParsFromDD.build(), SiStripFedCabling.buildFedCabling(), DTCombinatorialPatternReco4D.buildPhiSuperSegmentsCandidates(), DTMeantimerPatternReco4D.buildPhiSuperSegmentsCandidates(), DTCombinatorialPatternReco.buildPointsCollection(), DTCombinatorialExtendedPatternReco.buildPointsCollection(), FWDTSegmentProxyBuilder.buildViewType(), lumi::LumiSectionData.bunchCrossingInfo(), reco::FlavorHistoryEvent.cache(), ecaldqm::MESetNonObject.clone(), ecaldqm::MESetDet2D.clone(), ecaldqm::MESetProjection.clone(), ecaldqm::MESetDet1D.clone(), ecaldqm::MESetDet0D.clone(), ecaldqm::MESetTrend.clone(), ecaldqm::MESetEcal.clone(), ecaldqm::MESetMulti.clone(), ecaldqm::MESet.clone(), CmsShowMainFrame.CmsShowMainFrame(), EcalGlobalShowerContainmentCorrectionsVsEta::Coefficients.Coefficients(), EcalShowerContainmentCorrections::Coefficients.Coefficients(), SiStripProcessedRawDigiProducer.common_process(), CompoundPseudoJet.CompoundPseudoJet(), CompoundPseudoSubJet.CompoundPseudoSubJet(), DTCalibValidation.compute(), EcalDccWeightBuilder.computeAllWeights(), OMTFConfiguration.configure(), Measurement.construct(), DDHGCalModule.constructLayers(), DDHGCalModuleAlgo.constructLayers(), DDHGCalEEAlgo.constructLayers(), DDHGCalHEAlgo.constructLayers(), edm.copy_all(), copy_and_sort_vector(), MuonAlignment.copyAlignmentToSurvey(), PFBenchmarkAlgo.copyCollection(), Vispa.Main.Application.Application.copyEvent(), Measurement.copyMeas(), EcalDBCopy.copyToDB(), TruncatedPyramid.createCorners(), sistrip::DetSetVectorFiller< T, dsvIsSparse >.createDetSetVector(), align.createPoints(), TGeoMgrFromDdd.createShape(), TGeoFromDddService.createShape(), AlignableMuon.cscAlignmentErrorsExtended(), AlignableMuon.cscAlignments(), AlignableMuon.CSCChambers(), AlignableMuon.CSCEndcaps(), AlignableMuon.CSCRings(), AlignableMuon.CSCStations(), DetGeomDesc.deepComponents(), HGCalSimHitValidation.defineGeometry(), edm::StreamerInputSource.deserializeEvent(), magfieldparam::rz_poly.Diff(), reco::DiscretizedEnergyFlow.DiscretizedEnergyFlow(), FWLiteJobMetadataManager.doUpdate(), AlignableMuon.DTBarrel(), AlignableMuon.DTChambers(), AlignableMuon.DTStations(), AlignableMuon.DTWheels(), LHEWriter.endRun(), PhysicsTools::Interceptor.eval(), EventMsgBuilder.EventMsgBuilder(), eventSetupComponent(), DDTIDModulePosAlgo.execute(), DDPixBarStackTrigLayerAlgo.execute(), DDHGCalWaferAlgo.execute(), DDPixBarLayerAlgo.execute(), DDPixBarLayerUpgradeAlgo.execute(), DDPixFwdDiskAlgo.execute(), DDPixPhase1FwdDiskAlgo.execute(), DDTrackerLinearXY.execute(), DDTrackerXYZPosAlgo.execute(), DDTrackerZPosAlgo.execute(), DDPixBarLayerUpgradeAlgoCoverage.execute(), DDHCalFibreBundle.execute(), DDHCalLinearXY.execute(), DDTrackerAngular.execute(), DDTrackerAngularV1.execute(), DDLinear.execute(), DDAngular.execute(), DDTrackerRingAlgo.execute(), DDPixFwdBlades.execute(), reco::FFTJetProducerSummary.FFTJetProducerSummary(), Measurement.fillData(), MuonShowerInformationFiller.fillHitsByStation(), CSCHitFromStripOnly.fillPulseHeights(), DTNewROS8FileReader.fillRawData(), DTROS8FileReader.fillRawData(), DTDDUFileReader.fillRawData(), DTROS25FileReader.fillRawData(), DTSpyReader.fillRawData(), reco::modules::JetFlavourIdentifier.findCandidates(), DTCombinatorialPatternReco.findCompatibleHits(), DTCombinatorialExtendedPatternReco.findCompatibleHits(), CSCHitFromStripOnly.findHitOnStripPosition(), SequentialPartitionGenerator.first_part(), ecaldqm::binning.formAxis(), edm.get_underlying_safe(), EcalBarrelGeometry.getClosestCell(), ROOT::Math::Transform3DPJ.GetComponents(), DTTimeBoxFitter.getFitSeeds(), WatcherStreamFileReader.getInputFile(), HcalO2OManager.getListOfNewIovs_test(), MsgTools.getNames(), edm::ParameterSet.getParameterNamesForType(), PhysicsPerformancePayload.getRow(), METCorrectorParametersCollection.getSections(), JetCorrectorParametersCollection.getSections(), TrackTransformerForGlobalCosmicMuons.getTransientRecHits(), TrackTransformerForCosmicMuons.getTransientRecHits(), l1t::Parameter.getVector(), FedRawDataInputSource.grabNextJsonFile(), HcalLogicalMap.HcalLogicalMap(), EventMsgView.hltTriggerBits(), edm::RangeMap< int, std::vector< float >, edm::CopyPolicy< float > >.ids(), EcalSRCondTools.importParameterSet(), gen::JetMatchingAlpgen.init(), SimHitMatcher.init(), RecoIdealGeometry.insert(), edm::helper::Filler< Association< C > >.insert(), SiPixelClusterShapeCache.insert(), Phase2ITPixelClusterShapeCache.insert(), edmNew::DetSetVector< edm::Ref< edmNew::DetSetVector< SiStripCluster >, SiStripCluster, edmNew::DetSetVector< SiStripCluster >::FindForDetSetVector > >.insert(), magfieldparam::rz_poly.Int(), MatacqTBDataFormatter.interpretRawData(), MatacqDataFormatter.interpretRawData(), lhef::LHERunInfo.LHERunInfo(), CaloTowerHardcodeGeometryLoader.load(), HcalGeomParameters.loadGeometry(), HGCalGeomParameters.loadGeometryHexagon(), HGCalGeomParameters.loadGeometrySquare(), edm::MapOfVectors< std::string, AnalysisDescription * >.loadNext(), HGCalGeomParameters.loadWaferHexagon(), OptoScanTask.locateTicks(), MagFieldConfig.MagFieldConfig(), make_array(), CSCHitFromStripOnly.makeStripData(), reco::tau::RecoTauIsolationMasking.mask(), SiStripRecHitConverterAlgorithm.match(), GEMDigiMatcher.matchDigisToSimTrack(), GEMDigiMatcher.matchPadsToSimTrack(), GEMRecHitMatcher.matchRecHitsToSimTrack(), LHERunInfoProduct.mergeProduct(), MuRodBarrelLayer.MuRodBarrelLayer(), SequentialCombinationGenerator< T >.next_combi(), SequentialPartitionGenerator.next_part(), SequentialCombinationGenerator< T >.next_subset(), nonProducerComponent(), reco::tau::RecoTauPhotonFilter.operator()(), reco::tau::RecoTauElectronRejectionPlugin.operator()(), reco::TrackKinematics.operator+(), Json::ValueIteratorBase.operator-(), operator<<(), EcalGlobalShowerContainmentCorrectionsVsEta::Coefficients.operator=(), EcalShowerContainmentCorrections::Coefficients.operator=(), PhysicsTools::MVAComputer::DerivContext.output(), l1t::Parameter.Parameter(), edm.ParameterSet::getParameterNamesForType< std::vector< PdtEntry > >(), AlpgenHeader.parse(), PartitionGenerator.partitions(), CmsTrackerPhase1DiskBuilder.PhiPosNegSplit_innerOuter(), CmsTrackerPhase2TPDiskBuilder.PhiPosNegSplit_innerOuter(), fftjetcms::Polynomial.Polynomial(), DDHGCalModuleAlgo.positionSensitive(), DDHGCalModule.positionSensitive(), L1GtConditionEvaluation.print(), l1t::ConditionEvaluation.print(), GlobalObjectMap.print(), FillInfo.print(), L1GlobalTriggerObjectMap.print(), SiStripQuality.printActiveFedsInfo(), L1RCTProducer.printFedVector(), MagGeoBuilderFromDDD::volumeHandle.printUniqueNames(), sistrip::SpyDigiConverter.processFED(), MuonSeedOrcaPatternRecognition.produce(), EcalBasicClusterLocalContCorrectionsESProducer.produce(), DTRecSegment4DProducer.produce(), reco::tautools::CopyProducer< Collection >.produce(), pat::L1MuonMatcher.produce(), EcalGlobalShowerContainmentCorrectionsVsEtaESProducer.produce(), CSCTFPacker.produce(), EcalShowerContainmentCorrectionsESProducer.produce(), ReducedRecHitCollectionProducer.produce(), DTRecSegment2DExtendedProducer.produce(), EmbeddingLHEProducer.produce(), EcalTrivialConditionRetriever.produceEcalSamplesCorrelation(), EcalTrivialConditionRetriever.produceEcalTimeBiasCorrections(), HcalAlignmentEP.produceHcalAli(), edm::DataFrameContainer.push_back(), MuonDigiCollection< CSCDetId, CSCCLCTDigi >.put(), SurveyPxbImageReader< T >.read(), DTCalibrationMap.readConsts(), RunInfoRead.readData(), LmfSource.readEventWithinFile(), EcalTBReadout.readOut(), Model.readSystemDescription(), DTRefitAndCombineReco4D.refitSuperSegments(), own_ptr< std::ifstream >.release(), HGCalDDDConstants.rowColumnWafer(), run(), magfieldparam::rz_poly.rz_poly(), edm::Schedule.Schedule(), SeedTransformer.seedTransform(), edm::StreamerOutputModuleBase.serializeEvent(), edm::StreamerOutputModuleBase.serializeRegistry(), edm::DataFrameContainer.set_back(), lumi::LumiSectionData.setBunchCrossingData(), lumi::LumiSectionData.setHLTData(), lumi::LumiSectionData.setTriggerData(), PhysicsTools::MVAComputer.setup(), DQMStore.showDirStructure(), SiStripDetInfoFileReader.SiStripDetInfoFileReader(), SiStripHashedDetId.SiStripHashedDetId(), cond::SmallWORMDict.SmallWORMDict(), edm::DataFrameContainer.sort(), MuonDetLayerGeometry.sortLayers(), TrackerGeometryCompare.surveyToTracker(), edm::SystemTimeKeeper.SystemTimeKeeper(), topLevelPSet(), TrackerStablePhiSort(), reco::TrackExtrapolation.TrackExtrapolation(), TrackMVAClassifierBase.TrackMVAClassifierBase(), TrackTransformerForCosmicMuons.transform(), edm::TransientDataFrame< SIZE >.TransientDataFrame(), MuonTransientTrackingRecHit.transientHits(), MuonTrackLoader.unpackHit(), l1t::stage2.unpacking(), PrintGeomInfoAction.update(), PrintGeomMatInfo.update(), sistrip::RawToDigiUnpacker.update(), Version.Version(), AdaptiveVertexReconstructor.vertices(), HGCalDDDConstants.volumes(), DTCalibrationMap.writeConsts(), EcalDccWeightBuilder.writeWeightToRootFile(), and ext::basic_omemstream< Item_t, Traits_t, Allocator_t >.xsputn().

 def copy( args ):
     destDb = args.destDb
     destTag = args.destTag
     comment = args.comment
 
     datef = datetime.now()
     destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro"  }
     if destDb.lower() in destMap.keys():
         destDb = destMap[destDb.lower()]
     else:
         logger.error( 'Destination connection %s is not supported.' %destDb )
         return 
     # run the copy
     note = '"Importing data with O2O execution"'
     commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
     copyCommand = 'conddb %s' %commandOptions
     logger.info( 'Executing command: %s' %copyCommand )
     try:
         pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
         stdout = pipe.communicate()[0]
         print stdout
         retCode = pipe.returncode
         if retCode != 0:
             saveFileForImportErrors( datef )
         ret = retCode
     except Exception as e:
         ret = 1
         logger.error( str(e) )
     return ret
 

def popcon2dropbox.run ( args )

Definition at line 153 of file popcon2dropbox.py.

References checkFile(), copy(), and upload().

 def run( args ):
     if args.auth is not None and not args.auth=='':
         if auth_path_key in os.environ:
             logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
         else:
             logger.info("Setting the authentication path to %s in the environment." %args.auth)
             os.environ[auth_path_key]=args.auth
     if os.path.exists( '%s.db' %dbName ):
        logger.info("Removing files with name %s" %dbName )
        os.remove( '%s.db' %dbName )
     if os.path.exists( '%s.txt' %dbName ):
        os.remove( '%s.txt' %dbName )
     command = 'cmsRun %s ' %args.job_file
     command += ' destinationDatabase=%s' %args.destDb
     command += ' destinationTag=%s' %args.destTag
     command += ' 2>&1'
     pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
     stdout = pipe.communicate()[0]
     retCode = pipe.returncode
     print stdout
     logger.info('PopCon Analyzer return code is: %s' %retCode )
     if retCode!=0:
        logger.error( 'O2O job failed. Skipping upload.' )
        return retCode
 
     ret = checkFile()
     if ret < 0:
         return ret
     elif ret == 0:
         return 0
     if args.copy:
         return copy( args )
     return upload( args )

def popcon2dropbox.saveFileForImportErrors	(	datef,
		withMetadata = `False`
	)

Definition at line 61 of file popcon2dropbox.py.

Referenced by copy(), and upload().

 def saveFileForImportErrors( datef, withMetadata=False ):
     # save a copy of the files in case of upload failure...
     leafFolderName = datef.strftime(dateformatForFolder)
     fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
     if not os.path.exists(fileFolder):
         os.makedirs(fileFolder)
     df= '%s.db' %dbName
     dataDestFile = os.path.join( fileFolder, df)
     if not os.path.exists(dataDestFile):
         shutil.copy2(df, dataDestFile)
     if withMetadata:
         mf= '%s.txt' %dbName
         metadataDestFile = os.path.join( fileFolder, mf )
         if not os.path.exists(metadataDestFile):
             shutil.copy2(df, metadataDestFile)
     logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
     

def popcon2dropbox.upload ( args )

Definition at line 78 of file popcon2dropbox.py.

References saveFileForImportErrors(), and harvestTrackValidationPlots.str.

Referenced by run().

 def upload( args ):
     destDb = args.destDb
     destTag = args.destTag
     comment = args.comment
 
     datef = datetime.now()
 
     # first remove any existing metadata file...                                                                                                                                
     if os.path.exists( '%s.txt' %dbName ):
        logger.debug('Removing already existing file %s' %dbName)
        os.remove( '%s.txt' %dbName )
     
     # dump Metadata for the Upload
     uploadMd = {}
     uploadMd['destinationDatabase'] = destDb
     tags = {}
     tagInfo = {}
     tags[ destTag ] = tagInfo
     uploadMd['destinationTags'] = tags
     uploadMd['inputTag'] = destTag
     uploadMd['since'] = None
     datelabel = datef.strftime(dateformatForLabel)
     commentStr = ''
     if not comment is None:
        commentStr = comment
     uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
     with open( '%s.txt' %dbName, 'wb') as jf:
        jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
        jf.write('\n')
 
     # run the upload
     uploadCommand = 'uploadConditions.py %s' %dbName
     try:
        pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
        stdout = pipe.communicate()[0]
        print stdout
        retCode = pipe.returncode
        if retCode != 0:
            saveFileForImportErrors( datef, True )
        ret |= retCode
     except Exception as e:
        ret |= 1
        logger.error(str(e))
     return ret
 

Functions

Function Documentation