Functions
def	checkFile (dbName)

def	copy (args, dbName)

def	run (args)

def	saveFileForImportErrors (datef, dbName, withMetadata=False)

def	upload (args, dbName)

Function Documentation

def popcon2dropbox.checkFile ( dbName )

Definition at line 32 of file popcon2dropbox.py.

References str.

Referenced by run().

 def checkFile( dbName ):
     dbFileName = '%s.db' %dbName
     # check if the expected input file is there... 
     # exit code < 0 => error
     # exit code = 0 => skip
     # exit code = 1 => import                                                                                                                             
     if not os.path.exists( dbFileName ):
        logger.error('The file expected as an input %s has not been found.'%dbFileName )
        return -1
 
     empty = True
     try:
        dbcon = sqlite3.connect( dbFileName )
        dbcur = dbcon.cursor()
        dbcur.execute('SELECT * FROM IOV')
        rows = dbcur.fetchall()
        for r in rows:
           empty = False
        dbcon.close()
        if empty:
            logger.warning('The file expected as an input %s contains no data. The import will be skipped.'%dbFileName )
            return 0
        return 1
     except Exception as e:
        logger.error('Check on input data failed: %s' %str(e))
        return -2
 

def popcon2dropbox.copy	(	args,
		dbName
	)

Definition at line 122 of file popcon2dropbox.py.

References edm.print(), saveFileForImportErrors(), and str.

Referenced by __assert_no_abort(), PreMixingCaloParticleWorker.add(), DigiCollectionFP420.add(), AlignmentSurfaceDeformations.add(), GeometricTimingDet.addComponents(), GeometricDet.addComponents(), MatacqProducer.addMatacqData(), PreMixingDigiSimLinkWorker< DigiSimLinkCollection >.addPileups(), nanoaod::MergeableCounterTable.addVFloat(), nanoaod::MergeableCounterTable.addVInt(), algorithm(), AlignableExtras.alignmentErrors(), AlignableDet.alignmentErrors(), AlignableCSCEndcap.alignmentErrors(), AlignableDTBarrel.alignmentErrors(), AlignableComposite.alignmentErrors(), AlignableMuon.alignmentErrors(), AlignableTracker.alignmentErrors(), AlignableExtras.alignments(), AlignableDet.alignments(), AlignableCSCEndcap.alignments(), AlignableDTBarrel.alignments(), AlignableComposite.alignments(), AlignableMuon.alignments(), AlignableTracker.alignments(), LHEWriter.analyze(), DTResidualCalibration.analyze(), DTEfficiencyTask.analyze(), DTResolutionAnalysisTask.analyze(), FFTJetImageRecorder.analyze(), FFTJetPileupAnalyzer.analyze(), GeometryAligner.applyAlignments(), SegmentsTrackAssociator.associate(), GeometryAligner.attachSurfaceDeformations(), EmbeddingLHEProducer.beginRunProduce(), ecaldqm::MESetNonObject.book(), ecaldqm::MESetEcal.book(), L1ExtraDQM::L1ExtraMonElement< CollectionType >.bookhistograms(), MuonTransientTrackingRecHitBreaker.breakInSubRecHits(), HGCalGeometryLoader.build(), CSCGeometryParsFromDD.build(), SiStripFedCabling.buildFedCabling(), DTCombinatorialPatternReco4D.buildPhiSuperSegmentsCandidates(), DTMeantimerPatternReco4D.buildPhiSuperSegmentsCandidates(), DTCombinatorialPatternReco.buildPointsCollection(), DTCombinatorialExtendedPatternReco.buildPointsCollection(), FWDTSegmentProxyBuilder.buildViewType(), lumi::LumiSectionData.bunchCrossingInfo(), reco::FlavorHistoryEvent.cache(), ecaldqm::MESetNonObject.clone(), ecaldqm::MESetProjection.clone(), ecaldqm::MESetDet2D.clone(), ecaldqm::MESetDet1D.clone(), ecaldqm::MESetDet0D.clone(), ecaldqm::MESetTrend.clone(), ecaldqm::MESetEcal.clone(), ecaldqm::MESetMulti.clone(), ecaldqm::MESet.clone(), CmsShowMainFrame.CmsShowMainFrame(), EcalGlobalShowerContainmentCorrectionsVsEta::Coefficients.Coefficients(), EcalShowerContainmentCorrections::Coefficients.Coefficients(), SiStripProcessedRawDigiProducer.common_process(), CompoundPseudoJet.CompoundPseudoJet(), CompoundPseudoSubJet.CompoundPseudoSubJet(), DTCalibValidation.compute(), EcalDccWeightBuilder.computeAllWeights(), OMTFConfiguration.configure(), Measurement.construct(), DDAHcalModuleAlgo.constructLayers(), DDHGCalModule.constructLayers(), DDHGCalModuleAlgo.constructLayers(), DDHGCalTBModule.constructLayers(), DDHGCalEEAlgo.constructLayers(), DDHGCalHEAlgo.constructLayers(), edm.copy_all(), copy_and_sort_vector(), o2o_helper.copy_payload(), MuonAlignment.copyAlignmentToSurvey(), PFBenchmarkAlgo.copyCollection(), Vispa.Main.Application.Application.copyEvent(), edm::soa::Table< Args >.copyFromToWithResize(), Measurement.copyMeas(), EcalDBCopy.copyToDB(), TruncatedPyramid.createCorners(), sistrip::DetSetVectorFiller< T, dsvIsSparse >.createDetSetVector(), genericValidation.ValidationWithPlots.createPlottingScript(), align.createPoints(), TGeoMgrFromDdd.createShape(), TGeoFromDddService.createShape(), AlignableMuon.cscAlignmentErrorsExtended(), AlignableMuon.cscAlignments(), AlignableMuon.CSCChambers(), AlignableMuon.CSCEndcaps(), AlignableMuon.CSCRings(), AlignableMuon.CSCStations(), DetGeomDesc.deepComponents(), HGCalSimHitValidation.defineGeometry(), edm::StreamerInputSource.deserializeEvent(), magfieldparam::rz_poly.Diff(), reco::DiscretizedEnergyFlow.DiscretizedEnergyFlow(), genericValidation.ValidationWithComparison.doComparison(), FWLiteJobMetadataManager.doUpdate(), AlignableMuon.DTBarrel(), AlignableMuon.DTChambers(), AlignableMuon.DTStations(), AlignableMuon.DTWheels(), BeamFitter.dumpTxtFile(), empty(), MuonSeedOrcaPatternRecognition.endcapPatterns(), LHEWriter.endRun(), PhysicsTools::Interceptor.eval(), EventMsgBuilder.EventMsgBuilder(), eventSetupComponent(), DDHGCalWafer8.execute(), DDTIDModulePosAlgo.execute(), DDHGCalWafer.execute(), DDHGCalWaferAlgo.execute(), DDPixBarLayerUpgradeAlgo.execute(), DDPixFwdDiskAlgo.execute(), DDPixPhase1FwdDiskAlgo.execute(), DDTrackerLinearXY.execute(), DDPixBarLayerAlgo.execute(), DDTrackerXYZPosAlgo.execute(), DDTrackerZPosAlgo.execute(), DDHCalFibreBundle.execute(), DDHCalLinearXY.execute(), DDTrackerAngularV1.execute(), DDTrackerAngular.execute(), DDLinear.execute(), DDAngular.execute(), DDTrackerRingAlgo.execute(), DDPixFwdBlades.execute(), EMTFSubsystemCollector.extractPrimitives(), reco::FFTJetProducerSummary.FFTJetProducerSummary(), Measurement.fillData(), MuonShowerInformationFiller.fillHitsByStation(), CSCHitFromStripOnly.fillPulseHeights(), DTNewROS8FileReader.fillRawData(), DTROS8FileReader.fillRawData(), DTDDUFileReader.fillRawData(), DTROS25FileReader.fillRawData(), DTSpyReader.fillRawData(), CaloTruthAccumulator.finalizeEvent(), reco::modules::JetFlavourIdentifier.findCandidates(), DTCombinatorialPatternReco.findCompatibleHits(), DTCombinatorialExtendedPatternReco.findCompatibleHits(), CSCHitFromStripOnly.findHitOnStripPosition(), SequentialPartitionGenerator.first_part(), ecaldqm::binning.formAxis(), edm.get_underlying_safe(), EcalBarrelGeometry.getClosestCell(), genericValidation.GenericValidation.getCompareStrings(), ROOT::Math::Transform3DPJ.GetComponents(), EcalTrivialConditionRetriever.getEcalSimPulseShapeFromConfiguration(), l1t::stage2::GTCollections.getEGammas(), l1t::stage2::GTCollections.getEtSums(), DTTimeBoxFitter.getFitSeeds(), WatcherStreamFileReader.getInputFile(), l1t::stage2::GTCollections.getJets(), l1t::stage2::GTCollections.getMuons(), l1t::stage2::GMTCollections.getMuons(), MsgTools.getNames(), edm::ParameterSet.getParameterNamesForType(), PhysicsPerformancePayload.getRow(), METCorrectorParametersCollection.getSections(), JetCorrectorParametersCollection.getSections(), l1t::stage2::GTCollections.getTaus(), TrackTransformerForGlobalCosmicMuons.getTransientRecHits(), TrackTransformerForCosmicMuons.getTransientRecHits(), l1t::Parameter.getVector(), evf::EvFDaqDirector.grabNextJsonFileAndUnlock(), HcalLogicalMap.HcalLogicalMap(), EventMsgView.hltTriggerBits(), edm::RangeMap< int, std::vector< float >, edm::CopyPolicy< float > >.ids(), EcalSRCondTools.importParameterSet(), gen::JetMatchingAlpgen.init(), SimHitMatcher.init(), RecoIdealGeometry.insert(), edm::helper::Filler< Association< C > >.insert(), Phase2ITPixelClusterShapeCache.insert(), SiPixelClusterShapeCache.insert(), edmNew::DetSetVector< edm::Ref< edmNew::DetSetVector< SiStripCluster >, SiStripCluster, edmNew::DetSetVector< SiStripCluster >::FindForDetSetVector > >.insert(), magfieldparam::rz_poly.Int(), MatacqTBDataFormatter.interpretRawData(), MatacqDataFormatter.interpretRawData(), lhef::LHERunInfo.LHERunInfo(), CaloTowerHardcodeGeometryLoader.load(), HcalGeomParameters.loadGeometry(), HGCalGeomParameters.loadGeometryHexagon(), HGCalGeomParameters.loadGeometryHexagon8(), edm::MapOfVectors< std::string, AnalysisDescription * >.loadNext(), HGCalGeomParameters.loadWaferHexagon(), OptoScanTask.locateTicks(), MagFieldConfig.MagFieldConfig(), make_array(), DeepBoostedJetTagsProducer.make_inputs(), CSCHitFromStripOnly.makeStripData(), reco::tau::RecoTauIsolationMasking.mask(), SiStripRecHitConverterAlgorithm.match(), GEMDigiMatcher.matchDigisToSimTrack(), GEMDigiMatcher.matchPadsToSimTrack(), GEMRecHitMatcher.matchRecHitsToSimTrack(), LHERunInfoProduct.mergeProduct(), MTDStablePhiSort(), MTDTrayBarrelLayer.MTDTrayBarrelLayer(), MuRodBarrelLayer.MuRodBarrelLayer(), SequentialCombinationGenerator< T >.next_combi(), SequentialPartitionGenerator.next_part(), SequentialCombinationGenerator< T >.next_subset(), nonProducerComponent(), reco::tau::RecoTauPhotonFilter.operator()(), reco::tau::RecoTauElectronRejectionPlugin.operator()(), reco::TrackKinematics.operator+(), Json::ValueIteratorBase.operator-(), operator<<(), EcalGlobalShowerContainmentCorrectionsVsEta::Coefficients.operator=(), EcalShowerContainmentCorrections::Coefficients.operator=(), PhysicsTools::MVAComputer::DerivContext.output(), l1t::Parameter.Parameter(), edm.ParameterSet::getParameterNamesForType< std::vector< PdtEntry > >(), AlpgenHeader.parse(), PartitionGenerator.partitions(), PhiMemoryImage.PhiMemoryImage(), CmsTrackerPhase1DiskBuilder.PhiPosNegSplit_innerOuter(), CmsTrackerPhase2TPDiskBuilder.PhiPosNegSplit_innerOuter(), fftjetcms::Polynomial.Polynomial(), DDHGCalHEAlgo.positionMix(), DDAHcalModuleAlgo.positionSensitive(), DDHGCalModule.positionSensitive(), DDHGCalModuleAlgo.positionSensitive(), DDHGCalTBModule.positionSensitive(), DDHGCalEEAlgo.positionSensitive(), DDHGCalHEAlgo.positionSensitive(), edm::PreMixingModule.PreMixingModule(), L1GtConditionEvaluation.print(), l1t::ConditionEvaluation.print(), GlobalObjectMap.print(), L1GlobalTriggerObjectMap.print(), FillInfo.print(), LHCInfo.print(), SiStripQuality.printActiveFedsInfo(), L1RCTProducer.printFedVector(), MagGeoBuilderFromDDD::volumeHandle.printUniqueNames(), sistrip::SpyDigiConverter.processFED(), MuonSeedOrcaPatternRecognition.produce(), PFMuonUntagger.produce(), EcalBasicClusterLocalContCorrectionsESProducer.produce(), PFCandidateMuonUntagger.produce(), DTRecSegment4DProducer.produce(), reco::tautools::CopyProducer< Collection >.produce(), CSCTFPacker.produce(), EcalGlobalShowerContainmentCorrectionsVsEtaESProducer.produce(), pat::L1MuonMatcher.produce(), EcalShowerContainmentCorrectionsESProducer.produce(), ReducedRecHitCollectionProducer.produce(), DTRecSegment2DExtendedProducer.produce(), PFCandidateRecalibrator.produce(), EmbeddingLHEProducer.produce(), EcalTrivialConditionRetriever.produceEcalSamplesCorrelation(), EcalTrivialConditionRetriever.produceEcalTimeBiasCorrections(), HcalAlignmentEP.produceHcalAli(), FQueue< T >.push_back(), edm::DataFrameContainer.push_back(), MuonDigiCollection< uint16_t, GEMGEBStatusDigi >.put(), SurveyPxbImageReader< T >.read(), DTCalibrationMap.readConsts(), RunInfoRead.readData(), LmfSource.readEventWithinFile(), EcalTBReadout.readOut(), Model.readSystemDescription(), DTRefitAndCombineReco4D.refitSuperSegments(), edm::soa::Table< Args >.resizeFromTo(), PhiMemoryImage.rotl(), PhiMemoryImage.rotr(), HGCalDDDConstants.rowColumnWafer(), run(), SiStripClusterizerFromRaw.run(), magfieldparam::rz_poly.rz_poly(), edm::Schedule.Schedule(), SeedTransformer.seedTransform(), edm::StreamerOutputModuleBase.serializeEvent(), edm::StreamerOutputModuleBase.serializeRegistry(), edm::DataFrameContainer.set_back(), lumi::LumiSectionData.setBunchCrossingData(), l1t::stage2::EGammaUnpacker.setEGammaCopy(), l1t::stage2::EtSumUnpacker.setEtSumCopy(), l1t::stage2::EtSumUnpacker_0x10010057.setEtSumCopy(), lumi::LumiSectionData.setHLTData(), l1t::stage2::JetUnpacker.setJetCopy(), l1t::stage2::MuonUnpacker.setMuonCopy(), lumi::LumiSectionData.setTriggerData(), PhysicsTools::MVAComputer.setup(), DQMStore.showDirStructure(), SiStripDetInfoFileReader.SiStripDetInfoFileReader(), SiStripHashedDetId.SiStripHashedDetId(), cond::SmallWORMDict.SmallWORMDict(), edm::DataFrameContainer.sort(), MTDDetLayerGeometry.sortLayers(), MuonDetLayerGeometry.sortLayers(), SiStripZeroSuppression.storeCMN(), TrackerGeometryCompare.surveyToTracker(), edm::SystemTimeKeeper.SystemTimeKeeper(), topLevelPSet(), TrackerStablePhiSort(), reco::TrackExtrapolation.TrackExtrapolation(), TrackMVAClassifierBase.TrackMVAClassifierBase(), TrackTransformerForCosmicMuons.transform(), edm::TransientDataFrame< SIZE >.TransientDataFrame(), MuonTransientTrackingRecHit.transientHits(), MuonTrackLoader.unpackHit(), l1t::stage2.unpacking_bmtf(), PrintGeomInfoAction.update(), SimG4FluxProducer.update(), HGCPassive.update(), sistrip::RawToDigiUnpacker.update(), Version.Version(), AdaptiveVertexReconstructor.vertices(), HGCalDDDConstants.volumes(), DTCalibrationMap.writeConsts(), EcalDccWeightBuilder.writeWeightToRootFile(), and ext::basic_omemstream< Item_t, Traits_t, Allocator_t >.xsputn().

 def copy( args, dbName ):
     dbFileName = '%s.db' %dbName
     destDb = args.destDb
     destTag = args.destTag
     comment = args.comment
 
     datef = datetime.now()
     destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro"  }
     if destDb.lower() in destMap.keys():
         destDb = destMap[destDb.lower()]
     else:
         if destDb.startswith('sqlite'):
             destDb = destDb.split(':')[1]
         else:
             logger.error( 'Destination connection %s is not supported.' %destDb )
             return 
     # run the copy
     note = '"Importing data with O2O execution"'
     commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
     copyCommand = 'conddb %s' %commandOptions
     logger.info( 'Executing command: %s' %copyCommand )
     try:
         pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
         stdout = pipe.communicate()[0]
         print(stdout)
         retCode = pipe.returncode
         if retCode != 0:
             saveFileForImportErrors( datef, dbName )
         ret = retCode
     except Exception as e:
         ret = 1
         logger.error( str(e) )
     return ret
 

def popcon2dropbox.run ( args )

Definition at line 156 of file popcon2dropbox.py.

References checkFile(), copy(), edm.print(), and upload().

 def run( args ):
     
     dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
     dbFileName = '%s.db' %dbName
 
     if args.auth is not None and not args.auth=='':
         if auth_path_key in os.environ:
             logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
         else:
             logger.info("Setting the authentication path to %s in the environment." %args.auth)
             os.environ[auth_path_key]=args.auth
     if os.path.exists( '%s.db' %dbName ):
        logger.info("Removing files with name %s" %dbName )
        os.remove( '%s.db' %dbName )
     if os.path.exists( '%s.txt' %dbName ):
        os.remove( '%s.txt' %dbName )
     command = 'cmsRun %s ' %args.job_file
     command += ' targetFile=%s' %dbFileName
     command += ' destinationDatabase=%s' %args.destDb
     command += ' destinationTag=%s' %args.destTag
     command += ' 2>&1'
     pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
     stdout = pipe.communicate()[0]
     retCode = pipe.returncode
     print(stdout)
     logger.info('PopCon Analyzer return code is: %s' %retCode )
     if retCode!=0:
        logger.error( 'O2O job failed. Skipping upload.' )
        return retCode
 
     ret = checkFile( dbName )
     if ret > 0:
         if args.copy:
             ret = copy( args, dbName )
         else:
             ret = upload( args, dbName )
     os.remove( '%s.db' %dbName )
     return ret

def popcon2dropbox.saveFileForImportErrors	(	datef,
		dbName,
		withMetadata = `False`
	)

Definition at line 59 of file popcon2dropbox.py.

Referenced by copy(), and upload().

 def saveFileForImportErrors( datef, dbName, withMetadata=False ):
     # save a copy of the files in case of upload failure...
     leafFolderName = datef.strftime(dateformatForFolder)
     fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
     if not os.path.exists(fileFolder):
         os.makedirs(fileFolder)
     df= '%s.db' %dbName
     dataDestFile = os.path.join( fileFolder, df)
     if not os.path.exists(dataDestFile):
         shutil.copy2(df, dataDestFile)
     if withMetadata:
         mf= '%s.txt' %dbName
         metadataDestFile = os.path.join( fileFolder, mf )
         if not os.path.exists(metadataDestFile):
             shutil.copy2(df, metadataDestFile)
     logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
     

def popcon2dropbox.upload	(	args,
		dbName
	)

Definition at line 76 of file popcon2dropbox.py.

References edm.print(), saveFileForImportErrors(), and str.

Referenced by run().

 def upload( args, dbName ):
     destDb = args.destDb
     destTag = args.destTag
     comment = args.comment
 
     datef = datetime.now()
 
     # first remove any existing metadata file...                                                                                                                                
     if os.path.exists( '%s.txt' %dbName ):
        logger.debug('Removing already existing file %s' %dbName)
        os.remove( '%s.txt' %dbName )
     
     # dump Metadata for the Upload
     uploadMd = {}
     uploadMd['destinationDatabase'] = destDb
     tags = {}
     tagInfo = {}
     tags[ destTag ] = tagInfo
     uploadMd['destinationTags'] = tags
     uploadMd['inputTag'] = destTag
     uploadMd['since'] = None
     datelabel = datef.strftime(dateformatForLabel)
     commentStr = ''
     if not comment is None:
        commentStr = comment
     uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
     with open( '%s.txt' %dbName, 'wb') as jf:
        jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
        jf.write('\n')
 
     # run the upload
     uploadCommand = 'uploadConditions.py %s' %dbName
     ret = 0
     try:
        pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
        stdout = pipe.communicate()[0]
        print(stdout)
        retCode = pipe.returncode
        if retCode != 0:
            saveFileForImportErrors( datef, dbName, True )
        ret |= retCode
     except Exception as e:
        ret |= 1
        logger.error(str(e))
     return ret
 

Functions

Function Documentation