CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions
popcon2dropbox Namespace Reference

Functions

def checkFile (dbName)
 
def copy (args, dbName)
 
def run (args)
 
def saveFileForImportErrors (datef, dbName, withMetadata=False)
 
def upload (args, dbName)
 

Function Documentation

def popcon2dropbox.checkFile (   dbName)

Definition at line 32 of file popcon2dropbox.py.

References str.

Referenced by run().

32 def checkFile( dbName ):
33  dbFileName = '%s.db' %dbName
34  # check if the expected input file is there...
35  # exit code < 0 => error
36  # exit code = 0 => skip
37  # exit code = 1 => import
38  if not os.path.exists( dbFileName ):
39  logger.error('The file expected as an input %s has not been found.'%dbFileName )
40  return -1
41 
42  empty = True
43  try:
44  dbcon = sqlite3.connect( dbFileName )
45  dbcur = dbcon.cursor()
46  dbcur.execute('SELECT * FROM IOV')
47  rows = dbcur.fetchall()
48  for r in rows:
49  empty = False
50  dbcon.close()
51  if empty:
52  logger.warning('The file expected as an input %s contains no data. The import will be skipped.'%dbFileName )
53  return 0
54  return 1
55  except Exception as e:
56  logger.error('Check on input data failed: %s' %str(e))
57  return -2
58 
def checkFile(dbName)
#define str(s)
def popcon2dropbox.copy (   args,
  dbName 
)

Definition at line 122 of file popcon2dropbox.py.

References edm.print(), saveFileForImportErrors(), and str.

Referenced by DigiCollectionFP420.add(), PreMixingCaloParticleWorker.add(), AlignmentSurfaceDeformations.add(), GeometricTimingDet.addComponents(), GeometricDet.addComponents(), custom_jme_cff.AddJetID(), MatacqProducer.addMatacqData(), PreMixingDigiSimLinkWorker< DigiSimLinkCollection >.addPileups(), nanoaod::MergeableCounterTable.addVFloat(), nanoaod::MergeableCounterTable.addVFloatWithNorm(), nanoaod::MergeableCounterTable.addVInt(), algorithm(), AlignableExtras.alignmentErrors(), AlignableDet.alignmentErrors(), AlignableCSCEndcap.alignmentErrors(), AlignableDTBarrel.alignmentErrors(), AlignableComposite.alignmentErrors(), AlignableMuon.alignmentErrors(), AlignableTracker.alignmentErrors(), AlignableExtras.alignments(), AlignableDet.alignments(), AlignableCSCEndcap.alignments(), AlignableDTBarrel.alignments(), AlignableComposite.alignments(), AlignableMuon.alignments(), AlignableTracker.alignments(), LHEWriter.analyze(), DTResidualCalibration.analyze(), DTEfficiencyTask.analyze(), DTResolutionAnalysisTask.analyze(), FFTJetImageRecorder.analyze(), FFTJetPileupAnalyzer.analyze(), GeometryAligner.applyAlignments(), SegmentsTrackAssociator.associate(), GeometryAligner.attachSurfaceDeformations(), EmbeddingLHEProducer.beginRunProduce(), ecaldqm::MESetNonObject.book(), ecaldqm::MESetEcal.book(), L1ExtraDQM::L1ExtraMonElement< CollectionType >.bookhistograms(), MuonTransientTrackingRecHitBreaker.breakInSubRecHits(), HGCalGeometryLoader.build(), CSCGeometryParsFromDD.build(), SiStripFedCabling.buildFedCabling(), DTCombinatorialPatternReco4D.buildPhiSuperSegmentsCandidates(), DTMeantimerPatternReco4D.buildPhiSuperSegmentsCandidates(), DTCombinatorialPatternReco.buildPointsCollection(), DTCombinatorialExtendedPatternReco.buildPointsCollection(), FWDTSegmentProxyBuilder.buildViewType(), lumi::LumiSectionData.bunchCrossingInfo(), reco::FlavorHistoryEvent.cache(), cmsswFiletrace.cleanupenv(), cmsswConfigtrace.cleanupenv(), ecaldqm::MESetDet0D.clone(), ecaldqm::MESetNonObject.clone(), ecaldqm::MESetDet2D.clone(), ecaldqm::MESetProjection.clone(), ecaldqm::MESetMulti.clone(), ecaldqm::MESetDet1D.clone(), ecaldqm::MESetTrend.clone(), ecaldqm::MESetEcal.clone(), ecaldqm::MESet.clone(), CmsShowMainFrame.CmsShowMainFrame(), EcalGlobalShowerContainmentCorrectionsVsEta::Coefficients.Coefficients(), EcalShowerContainmentCorrections::Coefficients.Coefficients(), SiStripProcessedRawDigiProducer.common_process(), CompoundPseudoJet.CompoundPseudoJet(), CompoundPseudoSubJet.CompoundPseudoSubJet(), DTCalibValidationFromMuons.compute(), DTCalibValidation.compute(), EcalDccWeightBuilder.computeAllWeights(), OMTFConfiguration.configure(), Measurement.construct(), DDAHcalModuleAlgo.constructLayers(), DDHGCalModule.constructLayers(), DDHGCalModuleAlgo.constructLayers(), DDHGCalTBModule.constructLayers(), DDHGCalTBModuleX.constructLayers(), DDHGCalEEAlgo.constructLayers(), DDHGCalHEAlgo.constructLayers(), edm.copy_all(), copy_and_sort_vector(), o2o_helper.copy_payload(), MuonAlignment.copyAlignmentToSurvey(), PFBenchmarkAlgo.copyCollection(), Vispa.Main.Application.Application.copyEvent(), edm::soa::Table< Args >.copyFromToWithResize(), Measurement.copyMeas(), edm::shared_memory::WriteBuffer.copyToBuffer(), EcalDBCopy.copyToDB(), TruncatedPyramid.createCorners(), sistrip::DetSetVectorFiller< T, dsvIsSparse >.createDetSetVector(), genericValidation.ValidationWithPlots.createPlottingScript(), align.createPoints(), TGeoMgrFromDdd.createShape(), TGeoFromDddService.createShape(), AlignableMuon.cscAlignmentErrorsExtended(), AlignableMuon.cscAlignments(), AlignableMuon.CSCChambers(), AlignableMuon.CSCEndcaps(), AlignableMuon.CSCRings(), AlignableMuon.CSCStations(), HGCalSimHitValidation.defineGeometry(), edm::StreamerInputSource.deserializeEvent(), magfieldparam::rz_poly.Diff(), reco::DiscretizedEnergyFlow.DiscretizedEnergyFlow(), genericValidation.ValidationWithComparison.doComparison(), FWLiteJobMetadataManager.doUpdate(), AlignableMuon.DTBarrel(), AlignableMuon.DTChambers(), AlignableMuon.DTStations(), AlignableMuon.DTWheels(), BeamFitter.dumpTxtFile(), empty(), MuonSeedOrcaPatternRecognition.endcapPatterns(), LHEWriter.endRun(), PhysicsTools::Interceptor.eval(), EventMsgBuilder.EventMsgBuilder(), eventSetupComponent(), DDHGCalWafer8.execute(), DDHGCalWaferAlgo.execute(), DDHGCalWafer.execute(), DDTIDModulePosAlgo.execute(), DDPixBarLayerUpgradeAlgo.execute(), DDPixFwdDiskAlgo.execute(), DDPixPhase1FwdDiskAlgo.execute(), DDTrackerLinearXY.execute(), DDPixBarLayerAlgo.execute(), DDTrackerXYZPosAlgo.execute(), DDTrackerZPosAlgo.execute(), DDHCalFibreBundle.execute(), DDTrackerAngularV1.execute(), DDHCalLinearXY.execute(), DDTrackerAngular.execute(), DDLinear.execute(), DDAngular.execute(), DDTrackerRingAlgo.execute(), DDPixFwdBlades.execute(), EMTFSubsystemCollector.extractPrimitives(), reco::FFTJetProducerSummary.FFTJetProducerSummary(), Measurement.fillData(), MuonShowerInformationFiller.fillHitsByStation(), CSCHitFromStripOnly.fillPulseHeights(), DTNewROS8FileReader.fillRawData(), DTROS8FileReader.fillRawData(), DTDDUFileReader.fillRawData(), DTROS25FileReader.fillRawData(), DTSpyReader.fillRawData(), CaloTruthAccumulator.finalizeEvent(), reco::modules::JetFlavourIdentifier.findCandidates(), DTCombinatorialPatternReco.findCompatibleHits(), DTCombinatorialExtendedPatternReco.findCompatibleHits(), CSCHitFromStripOnly.findHitOnStripPosition(), SequentialPartitionGenerator.first_part(), ecaldqm::binning.formAxis(), edm.get_underlying_safe(), EcalBarrelGeometry.getClosestCell(), genericValidation.GenericValidation.getCompareStrings(), ROOT::Math::Transform3DPJ.GetComponents(), EcalTrivialConditionRetriever.getEcalSimPulseShapeFromConfiguration(), l1t::stage2::GTCollections.getEGammas(), l1t::stage2::GTCollections.getEtSums(), DTTimeBoxFitter.getFitSeeds(), WatcherStreamFileReader.getInputFile(), l1t::stage2::GTCollections.getJets(), l1t::stage2::GTCollections.getMuons(), l1t::stage2::GMTCollections.getMuons(), MsgTools.getNames(), edm::ParameterSet.getParameterNamesForType(), PhysicsPerformancePayload.getRow(), METCorrectorParametersCollection.getSections(), JetCorrectorParametersCollection.getSections(), l1t::stage2::GTCollections.getTaus(), TrackTransformerForGlobalCosmicMuons.getTransientRecHits(), TrackTransformerForCosmicMuons.getTransientRecHits(), l1t::Parameter.getVector(), GenWeightsTableProducer.globalBeginRun(), evf::EvFDaqDirector.grabNextJsonFileAndUnlock(), HcalLogicalMap.HcalLogicalMap(), EventMsgView.hltTriggerBits(), edm::RangeMap< det_id_type, edm::OwnVector< B > >.ids(), EcalSRCondTools.importParameterSet(), gen::JetMatchingAlpgen.init(), SimHitMatcher.init(), RecoIdealGeometry.insert(), edm::helper::Filler< Association< C > >.insert(), Phase2ITPixelClusterShapeCache.insert(), SiPixelClusterShapeCache.insert(), edmNew::DetSetVector< T >.insert(), magfieldparam::rz_poly.Int(), MatacqTBDataFormatter.interpretRawData(), MatacqDataFormatter.interpretRawData(), lhef::LHERunInfo.LHERunInfo(), CaloTowerHardcodeGeometryLoader.load(), HcalGeomParameters.loadGeometry(), HGCalGeomParameters.loadGeometryHexagon(), HGCalGeomParameters.loadGeometryHexagon8(), edm::MapOfVectors< std::string, AnalysisDescription * >.loadNext(), HGCalGeomParameters.loadWaferHexagon(), OptoScanTask.locateTicks(), MagFieldConfig.MagFieldConfig(), mixture.main(), make_array(), BoostedJetONNXJetTagsProducer.make_inputs(), ExternalLHEProducer.makeArgs(), CSCHitFromStripOnly.makeStripData(), reco::tau::RecoTauIsolationMasking.mask(), SiStripRecHitConverterAlgorithm.match(), GEMDigiMatcher.matchDigisToSimTrack(), GEMDigiMatcher.matchPadsToSimTrack(), GEMRecHitMatcher.matchRecHitsToSimTrack(), LHERunInfoProduct.mergeProduct(), MTDTrayBarrelLayer.MTDTrayBarrelLayer(), MuRodBarrelLayer.MuRodBarrelLayer(), SequentialCombinationGenerator< T >.next_combi(), SequentialPartitionGenerator.next_part(), SequentialCombinationGenerator< T >.next_subset(), nonProducerComponent(), reco::tau::RecoTauPhotonFilter.operator()(), reco::tau::RecoTauElectronRejectionPlugin.operator()(), reco::TrackKinematics.operator+(), Json::ValueIteratorBase.operator-(), operator<<(), EcalGlobalShowerContainmentCorrectionsVsEta::Coefficients.operator=(), EcalShowerContainmentCorrections::Coefficients.operator=(), PhysicsTools::MVAComputer::DerivContext.output(), l1t::Parameter.Parameter(), edm.ParameterSet::getParameterNamesForType< std::vector< PdtEntry > >(), AlpgenHeader.parse(), PartitionGenerator.partitions(), PhiMemoryImage.PhiMemoryImage(), CmsTrackerPhase2TPDiskBuilder.PhiPosNegSplit_innerOuter(), CmsTrackerPhase1DiskBuilder.PhiPosNegSplit_innerOuter(), fftjetcms::Polynomial.Polynomial(), DDHGCalHEAlgo.positionMix(), DDAHcalModuleAlgo.positionSensitive(), DDHGCalModule.positionSensitive(), DDHGCalModuleAlgo.positionSensitive(), DDHGCalTBModule.positionSensitive(), DDHGCalEEAlgo.positionSensitive(), DDHGCalTBModuleX.positionSensitive(), DDHGCalHEAlgo.positionSensitive(), edm::PreMixingModule.PreMixingModule(), L1GtConditionEvaluation.print(), l1t::ConditionEvaluation.print(), GlobalObjectMap.print(), L1GlobalTriggerObjectMap.print(), FillInfo.print(), LHCInfo.print(), SiStripQuality.printActiveFedsInfo(), L1RCTProducer.printFedVector(), MagGeoBuilderFromDDD::volumeHandle.printUniqueNames(), sistrip::SpyDigiConverter.processFED(), MuonSeedOrcaPatternRecognition.produce(), PFMuonUntagger.produce(), EcalBasicClusterLocalContCorrectionsESProducer.produce(), PFCandidateMuonUntagger.produce(), DTRecSegment4DProducer.produce(), reco::tautools::CopyProducer< Collection >.produce(), EcalGlobalShowerContainmentCorrectionsVsEtaESProducer.produce(), pat::L1MuonMatcher.produce(), CSCTFPacker.produce(), EcalShowerContainmentCorrectionsESProducer.produce(), ReducedRecHitCollectionProducer.produce(), UniqueMerger< InputCollection, OutputCollection, P >.produce(), DTRecSegment2DExtendedProducer.produce(), PFCandidateRecalibrator.produce(), EmbeddingLHEProducer.produce(), EcalTrivialConditionRetriever.produceEcalSamplesCorrelation(), EcalTrivialConditionRetriever.produceEcalTimeBiasCorrections(), HcalAlignmentEP.produceHcalAli(), FQueue< T >.push_back(), edm::DataFrameContainer.push_back(), MuonDigiCollection< RPCDetId, RPCDigi >.put(), PYBIND11_MODULE(), SurveyPxbImageReader< T >.read(), DTCalibrationMap.readConsts(), RunInfoRead.readData(), LmfSource.readEventWithinFile(), EcalTBReadout.readOut(), Model.readSystemDescription(), DTRefitAndCombineReco4D.refitSuperSegments(), edm::soa::Table< Args >.resizeFromTo(), PhiMemoryImage.rotl(), PhiMemoryImage.rotr(), HGCalDDDConstants.rowColumnWafer(), run(), SiStripClusterizerFromRaw.run(), magfieldparam::rz_poly.rz_poly(), edm::Schedule.Schedule(), SeedTransformer.seedTransform(), edm::StreamerOutputModuleCommon.serializeEvent(), edm::StreamerOutputModuleCommon.serializeRegistry(), edm::DataFrameContainer.set_back(), lumi::LumiSectionData.setBunchCrossingData(), l1t::stage2::EGammaUnpacker.setEGammaCopy(), l1t::stage2::EtSumUnpacker_0x10010057.setEtSumCopy(), l1t::stage2::EtSumUnpacker.setEtSumCopy(), EGFull5x5ShowerShapeModifierFromValueMaps.setEvent(), EGExtraInfoModifierFromValueMaps< MapType, OutputType >.setEvent(), lumi::LumiSectionData.setHLTData(), l1t::stage2::JetUnpacker.setJetCopy(), l1t::stage2::MuonUnpacker.setMuonCopy(), lumi::LumiSectionData.setTriggerData(), PhysicsTools::MVAComputer.setup(), DQMStore.showDirStructure(), SiStripDetInfoFileReader.SiStripDetInfoFileReader(), SiStripHashedDetId.SiStripHashedDetId(), cond::SmallWORMDict.SmallWORMDict(), edm::DataFrameContainer.sort(), MTDDetLayerGeometry.sortLayers(), MuonDetLayerGeometry.sortLayers(), SiStripZeroSuppression.storeCMN(), TrackerGeometryCompare.surveyToTracker(), edm::SystemTimeKeeper.SystemTimeKeeper(), topLevelPSet(), trackerStablePhiSort(), reco::TrackExtrapolation.TrackExtrapolation(), TrackMVAClassifierBase.TrackMVAClassifierBase(), TrackTransformerForCosmicMuons.transform(), edm::TransientDataFrame< SIZE >.TransientDataFrame(), MuonTransientTrackingRecHit.transientHits(), MuonTrackLoader.unpackHit(), l1t::stage2.unpacking_bmtf(), PrintGeomInfoAction.update(), SimG4FluxProducer.update(), HGCPassive.update(), sistrip::RawToDigiUnpacker.update(), Version.Version(), AdaptiveVertexReconstructor.vertices(), HGCalDDDConstants.volumes(), DTCalibrationMap.writeConsts(), EcalDccWeightBuilder.writeWeightToRootFile(), and ext::basic_omemstream< Item_t, Traits_t, Allocator_t >.xsputn().

122 def copy( args, dbName ):
123  dbFileName = '%s.db' %dbName
124  destDb = args.destDb
125  destTag = args.destTag
126  comment = args.comment
127 
128  datef = datetime.now()
129  destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro" }
130  if destDb.lower() in destMap.keys():
131  destDb = destMap[destDb.lower()]
132  else:
133  if destDb.startswith('sqlite'):
134  destDb = destDb.split(':')[1]
135  else:
136  logger.error( 'Destination connection %s is not supported.' %destDb )
137  return
138  # run the copy
139  note = '"Importing data with O2O execution"'
140  commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
141  copyCommand = 'conddb %s' %commandOptions
142  logger.info( 'Executing command: %s' %copyCommand )
143  try:
144  pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
145  stdout = pipe.communicate()[0]
146  print(stdout)
147  retCode = pipe.returncode
148  if retCode != 0:
149  saveFileForImportErrors( datef, dbName )
150  ret = retCode
151  except Exception as e:
152  ret = 1
153  logger.error( str(e) )
154  return ret
155 
def saveFileForImportErrors(datef, dbName, withMetadata=False)
def copy(args, dbName)
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
#define str(s)
def popcon2dropbox.run (   args)

Definition at line 156 of file popcon2dropbox.py.

References checkFile(), copy(), edm.print(), and upload().

156 def run( args ):
157 
158  dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
159  dbFileName = '%s.db' %dbName
160 
161  if args.auth is not None and not args.auth=='':
162  if auth_path_key in os.environ:
163  logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
164  else:
165  logger.info("Setting the authentication path to %s in the environment." %args.auth)
166  os.environ[auth_path_key]=args.auth
167  if os.path.exists( '%s.db' %dbName ):
168  logger.info("Removing files with name %s" %dbName )
169  os.remove( '%s.db' %dbName )
170  if os.path.exists( '%s.txt' %dbName ):
171  os.remove( '%s.txt' %dbName )
172  command = 'cmsRun %s ' %args.job_file
173  command += ' targetFile=%s' %dbFileName
174  command += ' destinationDatabase=%s' %args.destDb
175  command += ' destinationTag=%s' %args.destTag
176  command += ' 2>&1'
177  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
178  stdout = pipe.communicate()[0]
179  retCode = pipe.returncode
180  print(stdout)
181  logger.info('PopCon Analyzer return code is: %s' %retCode )
182  if retCode!=0:
183  logger.error( 'O2O job failed. Skipping upload.' )
184  return retCode
185 
186  ret = checkFile( dbName )
187  if ret > 0:
188  if args.copy:
189  ret = copy( args, dbName )
190  else:
191  ret = upload( args, dbName )
192  os.remove( '%s.db' %dbName )
193  return ret
194 
def copy(args, dbName)
def upload(args, dbName)
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
def checkFile(dbName)
def popcon2dropbox.saveFileForImportErrors (   datef,
  dbName,
  withMetadata = False 
)

Definition at line 59 of file popcon2dropbox.py.

Referenced by copy(), and upload().

59 def saveFileForImportErrors( datef, dbName, withMetadata=False ):
60  # save a copy of the files in case of upload failure...
61  leafFolderName = datef.strftime(dateformatForFolder)
62  fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
63  if not os.path.exists(fileFolder):
64  os.makedirs(fileFolder)
65  df= '%s.db' %dbName
66  dataDestFile = os.path.join( fileFolder, df)
67  if not os.path.exists(dataDestFile):
68  shutil.copy2(df, dataDestFile)
69  if withMetadata:
70  mf= '%s.txt' %dbName
71  metadataDestFile = os.path.join( fileFolder, mf )
72  if not os.path.exists(metadataDestFile):
73  shutil.copy2(df, metadataDestFile)
74  logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
75 
def saveFileForImportErrors(datef, dbName, withMetadata=False)
def popcon2dropbox.upload (   args,
  dbName 
)

Definition at line 76 of file popcon2dropbox.py.

References edm.print(), saveFileForImportErrors(), and str.

Referenced by run().

76 def upload( args, dbName ):
77  destDb = args.destDb
78  destTag = args.destTag
79  comment = args.comment
80 
81  datef = datetime.now()
82 
83  # first remove any existing metadata file...
84  if os.path.exists( '%s.txt' %dbName ):
85  logger.debug('Removing already existing file %s' %dbName)
86  os.remove( '%s.txt' %dbName )
87 
88  # dump Metadata for the Upload
89  uploadMd = {}
90  uploadMd['destinationDatabase'] = destDb
91  tags = {}
92  tagInfo = {}
93  tags[ destTag ] = tagInfo
94  uploadMd['destinationTags'] = tags
95  uploadMd['inputTag'] = destTag
96  uploadMd['since'] = None
97  datelabel = datef.strftime(dateformatForLabel)
98  commentStr = ''
99  if not comment is None:
100  commentStr = comment
101  uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
102  with open( '%s.txt' %dbName, 'wb') as jf:
103  jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
104  jf.write('\n')
105 
106  # run the upload
107  uploadCommand = 'uploadConditions.py %s' %dbName
108  ret = 0
109  try:
110  pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
111  stdout = pipe.communicate()[0]
112  print(stdout)
113  retCode = pipe.returncode
114  if retCode != 0:
115  saveFileForImportErrors( datef, dbName, True )
116  ret |= retCode
117  except Exception as e:
118  ret |= 1
119  logger.error(str(e))
120  return ret
121 
def saveFileForImportErrors(datef, dbName, withMetadata=False)
def upload(args, dbName)
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
#define str(s)