CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsLHEtoEOSManager Namespace Reference

Functions

def articleExist
 
def fileUpload
 
def findXrdDir
 
def lastArticle
 

Variables

string __version__ = "$Revision: 1.13 $"
 
string action = 'store_true'
 
string default = ''
 
string defaultEOScpCommand = 'xrdcp -np '
 
string defaultEOSfeCommand = 'xrd eoscms existfile '
 
string defaultEOSlistCommand = 'xrd eoscms dirlist '
 
string defaultEOSLoadPath = 'root://eoscms/'
 
string defaultEOSmkdirCommand = 'xrd eoscms mkdir '
 
string defaultEOSRootPath = '/eos/cms/store/lhe'
 
string dest = 'fileList'
 
tuple exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])
 
tuple exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE)
 
tuple exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])
 
tuple exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])
 
tuple exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
 
tuple exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])
 
tuple f = open(options.files_from)
 
tuple getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE)
 
string help = 'LHE local file list to be uploaded, separated by ","'
 
tuple intCode = exeCheckIntegrity.wait()
 
tuple l = l.strip()
 
string listPath = defaultEOSRootPath+'/'
 
int newArt = 0
 
tuple oldArt = lastArticle()
 
tuple parser = optparse.OptionParser(usage)
 
 reallyDoIt = notoptions.dryRun
 
string theCheckIntegrityCommand = 'xmllint -noout '
 
list theCheckSumList = []
 
string theCommand = defaultEOSmkdirCommand+' '
 
list theCompressedFilesList = []
 
string theCompressionCommand = 'xz '
 
list theList = []
 
string uploadPath = ''
 
string usage = 'cmsLHEtoEOSManager.py <options>'
 

Function Documentation

def cmsLHEtoEOSManager.articleExist (   artId)

Definition at line 27 of file cmsLHEtoEOSManager.py.

References findXrdDir().

27 
28 def articleExist(artId):
29 
30  itExists = False
31  theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
32  dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
33  for line in dirList.stdout.readlines():
34  if findXrdDir(line) == str(artId):
35  itExists = True
36 
37  return itExists
def cmsLHEtoEOSManager.fileUpload (   uploadPath,
  lheList,
  checkSumList,
  reallyDoIt 
)

Definition at line 54 of file cmsLHEtoEOSManager.py.

54 
55 def fileUpload(uploadPath,lheList, checkSumList, reallyDoIt):
56 
57  inUploadScript = ''
58  index = 0
59  for f in lheList:
60  realFileName = f.split('/')[-1]
61  # Check the file existence
62  newFileName = uploadPath+'/'+str(realFileName)
63  addFile = True
64  additionalOption = ''
65  theCommand = defaultEOSfeCommand+' '+newFileName
66  exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
67  result = exeFullList.stdout.readlines()
68  if result[0].rstrip('\n') == 'The file exists.':
69  addFile = False
70  print 'File '+newFileName+' already exists: do you want to overwrite? [y/n]'
71  reply = raw_input()
72  if reply == 'y' or reply == 'Y':
73  addFile = True
74  additionalOption = ' -f '
75  print ''
76  print 'Overwriting file '+newFileName+'\n'
77  # add the file
78  if addFile:
79 # print 'Adding file '+str(f)+'\n'
80  inUploadScript = defaultEOScpCommand + additionalOption + ' ' + str(f) + ' ' + defaultEOSLoadPath+uploadPath + '/' + str(realFileName)
81  print 'Uploading file %s...' % str(f)
82  if reallyDoIt:
83  exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
84  exeRealUpload.communicate()
85  eosCheckSumCommand = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select find --checksum ' + uploadPath + '/' + str(realFileName) + ' | awk \'{print $2}\' | cut -d= -f2'
86  exeEosCheckSum = subprocess.Popen(eosCheckSumCommand ,shell=True, stdout=subprocess.PIPE)
87  EosCheckSum = exeEosCheckSum.stdout.read()
88  assert exeEosCheckSum.wait() == 0
89  # print 'checksum: eos = ' + EosCheckSum + 'orig file = ' + checkSumList[index] + '\n'
90  if checkSumList[index] not in EosCheckSum:
91  print 'WARNING! The checksum for file ' + str(realFileName) + ' in EOS\n'
92  print EosCheckSum + '\n'
93  print 'does not match the checksum of the original one\n'
94  print checkSumList[index] + '\n'
95  print 'please try to re-upload file ' + str(realFileName) + ' to EOS.\n'
96  else:
97  print 'Checksum OK for file ' + str(realFileName)
98  index = index+1
99 
100 # launch the upload shell script
101 
102 # print '\n Launching upload script \n'+inUploadScript+'\n at '+time.asctime(time.localtime(time.time()))+' ...\n'
103 # if reallyDoIt:
104 # exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
105 # exeRealUpload.communicate()
106  print '\n Upload ended at '+time.asctime(time.localtime(time.time()))
def cmsLHEtoEOSManager.findXrdDir (   theDirRecord)

Definition at line 19 of file cmsLHEtoEOSManager.py.

References split.

Referenced by articleExist(), and lastArticle().

19 
20 def findXrdDir(theDirRecord):
21 
22  elements = theDirRecord.split(' ')
23  if len(elements) > 1:
24  return elements[-1].rstrip('\n').split('/')[-1]
25  else:
26  return None
double split
Definition: MVATrainer.cc:139
def cmsLHEtoEOSManager.lastArticle ( )

Definition at line 38 of file cmsLHEtoEOSManager.py.

References findXrdDir(), and bookConverter.max.

38 
39 def lastArticle():
40 
41  artList = [0]
42 
43  theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
44  dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
45  for line in dirList.stdout.readlines():
46  try:
47  if line.rstrip('\n') != '':
48  artList.append(int(findXrdDir(line)))
49  except:
50  break
51 
52  return max(artList)
53 

Variable Documentation

string cmsLHEtoEOSManager.__version__ = "$Revision: 1.13 $"

Definition at line 5 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.action = 'store_true'

Definition at line 126 of file cmsLHEtoEOSManager.py.

int cmsLHEtoEOSManager.default = ''

Definition at line 118 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOScpCommand = 'xrdcp -np '

Definition at line 17 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSfeCommand = 'xrd eoscms existfile '

Definition at line 16 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSlistCommand = 'xrd eoscms dirlist '

Definition at line 14 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSLoadPath = 'root://eoscms/'

Definition at line 13 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSmkdirCommand = 'xrd eoscms mkdir '

Definition at line 15 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSRootPath = '/eos/cms/store/lhe'

Definition at line 12 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.dest = 'fileList'

Definition at line 119 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])

Definition at line 213 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE)

Definition at line 228 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])

Definition at line 222 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])

Definition at line 276 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)

Definition at line 263 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])

Definition at line 246 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.f = open(options.files_from)

Definition at line 188 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE)

Definition at line 229 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.help = 'LHE local file list to be uploaded, separated by ","'

Definition at line 117 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.intCode = exeCheckIntegrity.wait()

Definition at line 214 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.l = l.strip()

Definition at line 192 of file cmsLHEtoEOSManager.py.

Referenced by DTSuperLayer.add(), edm::service::ELlimitsTable.add(), DTGeometry.add(), ReferenceTrajectory.addMaterialEffectsBp(), ReferenceTrajectory.addMaterialEffectsBrl(), ReferenceTrajectory.addMaterialEffectsCov(), SiStripHitEffFromCalibTree.algoAnalyze(), AlignmentMonitorSurvey.AlignmentMonitorSurvey(), AllocNetwork(), EgammaBasicClusters.analyze(), MonitorElementsDb.analyze(), MCEfficiencyAnalyzer.analyze(), DQMHcalIsoTrackAlCaReco.analyze(), ContainmentCorrectionAnalyzer.analyze(), EnergyScaleAnalyzer.analyze(), cms::Analyzer_minbias.analyze(), PhotonValidator.analyze(), HLTScalers.analyze(), ValidationHcalIsoTrackAlCaReco.analyze(), HcalDetDiagPedestalMonitor.analyze(), EgammaObjects.analyzeElectrons(), EgammaObjects.analyzePhotons(), tauImpactParameter::LagrangeMultipliersFitter.applyLagrangianConstraints(), gbl::BorderedBandMatrix.bandOfAVAT(), BeamHaloNavigationSchool.BeamHaloNavigationSchool(), cms::Analyzer_minbias.beginJob(), BestTracks(), AlignmentMonitorSurvey.book(), ElectronTagProbeAnalyzer.bookHistograms(), ElectronAnalyzer.bookHistograms(), DQMHcalIsoTrackAlCaReco.bookHistograms(), PSMonitor.bookHistograms(), L1TRPCTF.bookHistograms(), PrimaryVertexAnalyzer4PUSlimmed.bookHistograms(), ActionInitialization.Build(), FWTrackResidualDetailView.build(), AlignableBuilder.buildAll(), DeDxTools.buildDiscrimMap(), IsoTrig.chgIsolation(), DTTFFEDReader.clear(), CmsShowCommon.CmsShowCommon(), CompositeLogicalTrajectoryFilter.CompositeLogicalTrajectoryFilter(), Conv.convolution(), dtCalibration::DTT0FEBPathCorrection.correction(), PSFitter::HybridMinimizer.Correlation(), PSFitter::HybridMinimizer.CovMatrix(), pftools::SpaceManager.createCalibrators(), MuonSeedFromRecHits.createSeed(), MuonSeedCreator.createSeed(), BTagLikeDeDxDiscriminator.dedx(), HitDoublets.detLayer(), cscdqm::StripClusterFinder.DoAction(), l1t::Stage1Layer2EtSumAlgorithmImpHW.doSumAndMET(), l1t::Stage1Layer2EtSumAlgorithmImpHI.doSumAndMET(), npstat::ArrayND< Numeric, StackLen, StackDim >.dotProductLoop(), JetMETHLTOfflineClient.dqmEndJob(), EcalDisplaysByEvent.drawEELines(), RPCDigiL1Link.empty(), TowerBlockFormatter.EndEvent(), cms::Analyzer_minbias.endJob(), LumiCalculator.endLuminosityBlock(), edm::OwnArray< T, MAX_SIZE, P >.erase(), ESPedestalTask.ESPedestalTask(), L1ExtraParticleMapProd.evaluateForwardRapidityGap(), VoronoiAlgorithm.event_fourier(), FWGUIManager.exportAllViews(), TFParams.f3deg(), LMFUnique.fetchAll(), LMFCorrCoefDat.fetchBetween(), LMFSeqDat.fetchByRunIOV(), LMFSeqDat.fetchByRunNumber(), LMFRunIOV.fetchBySequence(), HLTOfflineDQMTopDiLepton::MonitorDiLepton.fill(), CSCChamberIndexValues.fillChamberIndex(), CSCChamberMapValues.fillChamberMap(), CSCCrateMapValues.fillCrateMap(), CSCDDUMapValues.fillDDUMap(), ExpressLumiProducer.fillLSCache(), LumiProducer.fillLSCache(), emtf::Node.filterEventsToDaughters(), SurveyAlignmentPoints.findAlignPars(), SurveyAlignmentSensor.findAlignPars(), MuonCkfTrajectoryBuilder.findCompatibleMeasurements(), CSCHitFromStripOnly.findMaxima(), HLTPerformanceInfo.findPath(), CSCChamberFitter.fit(), cond::persistency::IOVEditor.flush(), TSFit.fpol3dg(), cond::SmallWORMDict::IterHelp.frame(), CmsShowNavigator.frameTitle(), GeometricSearchTracker.GeometricSearchTracker(), SiStripLayoutParser.getAllLayouts(), SiPixelLayoutParser.getAllLayouts(), MEEEGeom.getBoundary(), cond::persistency.getConnectionParams(), npstat::BoxNDScanner< Numeric >.getCoords(), PSFitter::HybridMinimizer.GetCovMatrix(), HcalDDDRecConstants.getEtaBins(), MEEEGeom.getGraphBoundary(), PSFitter::HybridMinimizer.GetHessianMatrix(), npstat::ArrayNDScanner.getIndex(), npstat::BoxNDScanner< Numeric >.getIndex(), npstat::GridAxis.getInterval(), LMFDat.getKeyList(), hcaldqm::mapper::HashMapper.getLinearHashType(), LMFDat.getLogicIds(), ValidHitPairFilter.getMissingLayers(), ConstantStepOdeSolver.getPeakTime(), TangentCircle.getPosition(), RPCReadOutMapping.getRAWSpecForCMSChamberSrip(), hcaldqm::quantity::LumiSection.getValue(), PythonParameterSet.getVPSet(), EcalTrivialConditionRetriever.getWeightsFromConfiguration(), align::tpb.halfBarrelNumber(), align::tib.halfShellNumber(), HBHELinearMap.HBHELinearMap(), HcalDDDSimConstants.HcalCellTypes(), CSCChamberFitter.hessian(), QualityCutsAnalyzer::histogram_element_t.histogram_element_t(), HitDoublets.hit(), HLTMuonL3PreFilter.hltFilter(), MSLayersAtAngle.init(), reco::GhostTrackPrediction.init(), cond::EmptyIOVSource.initialize(), SurveyAlignmentAlgorithm.initialize(), HcalDDDSimConstants.initialize(), edm::StreamSchedule.initializeEarlyDelete(), SherpaHadronizer.initializeForInternalPartons(), PixelRecoLineRZ.initTIP2(), popcon::EcalDCSHandler.insertHVDataSetToOffline(), gbl::VSymMatrix.invert(), edm.isFinite(), reco::TrackProbabilityTagInfo.jetProbability(), TemplatedJetProbabilityComputer< Container, Base >.jetProbability(), TemplatedJetBProbabilityComputer< Container, Base >.jetProbability(), align::tpb.ladderNumber(), langaupro(), HDQMUtil.langaupro(), LevelBaseNumber.LevelBaseNumber(), cond::time.limitedIOVSyncValue(), npstat::GridAxis.linearInterval(), MuonNavigationSchool.linkEndcapLayers(), fit::Likelihood< Sample, PDF, Yield >.log(), fit::Likelihood< Sample, PDF, NoExtendedLikelihood >.log(), main(), npstat::ArrayND< Numeric, StackLen, StackDim >.makeCopulaSteps(), fireworks_root_gui.makeLabel(), AlignableBuilder.maxComponent(), TkGluedMeasurementDet.measurements(), KDTreeLinkerAlgo< DATA >.medianSearch(), npstat::ArrayND< Numeric, StackLen, StackDim >.multiMirror(), MuonGeometryArrange.MuonGeometryArrange(), MuonSimHitsValidAnalyzer.MuonSimHitsValidAnalyzer(), SequentialCombinationGenerator< T >.next_combination(), pat.operator&(), TSCBLBuilderWithPropagator.operator()(), gbl::VSymMatrix.operator*(), operator==(), pat.operator^(), pat.operator|(), Signal.operator||(), logintpack.pack8log(), logintpack.pack8logCeil(), logintpack.pack8logClosed(), HitDoublets.phi(), HIPplots.plotAlignableChiSquare(), FWPopupMenu.PoppedDown(), FWPopupMenu.PoppedUp(), PrimaryVertexAnalyzer4PUSlimmed.PrimaryVertexAnalyzer4PUSlimmed(), cscdqm::StripClusterFinder.printClusters(), MuonNavigationPrinter.printLayers(), hitfit::Defaults_Textrep.process_args(), HcalNZSMonitor.processEvent(), StEvtSolutionMaker.produce(), CastorClusterProducer.produce(), cms::CosmicTrackFinder.produce(), SiPixelTemplate2D.pushfile(), SiStripTemplate.pushfile(), SiPixelTemplate.pushfile(), hitfit::Defaults_Textrep.read_file(), ReadPatterns(), PixelToFEDAssociateFromAscii.readRange(), dqmservices::DQMStreamerReader.reset_(), PFPhotonAlgo.RunPFPhoton(), HitDoublets.rv(), TrackerMap.save(), TrackerMap.save_as_fectrackermap(), TrackerMap.save_as_fedtrackermap(), TrackerMap.save_as_HVtrackermap(), TrackerMap.save_as_psutrackermap(), emtf::Tree.saveToXMLRecursive(), JsonOutputProducer::JsonEventState.serialise(), BinomialInterval.set(), FWProxyBuilderBase.setInteractionList(), FWGenParticleLegoProxyBuilder.setItem(), FWTrackingParticleProxyBuilder.setItem(), FWVertexProxyBuilder.setItem(), FWVertexCandidateProxyBuilder.setItem(), gen::TauolappInterface.SetLHE(), FWBoxRecHit.setLine(), FWPFLegoRecHit.setLine(), reco::PFBlock.setLink(), reco::PFDisplacedVertexCandidate.setLink(), CachedTrajectory.setMaxDetectorLength(), CachedTrajectory.setMaxHOLength(), CachedTrajectory.setMinDetectorLength(), Plane.setPosPrec(), cond::EmptyIOVSource.setRunAndEventInfo(), FWJetProxyBuilder.setTextPos(), MuonDetLayerGeometry.sortLayers(), StringBasedNTupler.StringBasedNTupler(), align::tib.stringNumber(), cscdqm::StripClusterFinder.StripClusterFinder(), VoronoiAlgorithm.subtract_momentum(), GBRTree.TerminalIndex(), HIPAlignmentAlgorithm.terminate(), LutXml.test_access(), cond::time.toIOVSyncValue(), TrackerGeometryCompare.TrackerGeometryCompare(), TrackMultiSelector.TrackMultiSelector(), funct.trapezoid_integral(), dqmservices::TriggerSelector::TreeElement.TreeElement(), logintpack.unpack8log(), logintpack.unpack8logClosed(), CSCEventData.unpack_data(), gen::TauolappInterface.update_particles(), FWLegoEvePFCandidate.updateScale(), FWPFRhoPhiRecHit.updateScale(), FWLegoCandidate.updateScale(), VarSplitter.VarSplitter(), reco::TrackGhostTrackState.vertexStateOnGhostTrack(), reco::TrackGhostTrackState.vertexStateOnMeasurement(), VVIObj.VVIObj(), sistripvvi::VVIObj.VVIObj(), VVIObjF.VVIObjF(), SimpleNavigableLayer.wellInside(), DCULVRVoltagesDat.writeArrayDB(), HitDoublets.x(), cms::concurrency.xercesInitialize(), cms::concurrency.xercesTerminate(), HitDoublets.y(), HitDoublets.z(), and GeometricSearchTracker.~GeometricSearchTracker().

string cmsLHEtoEOSManager.listPath = defaultEOSRootPath+'/'

Definition at line 261 of file cmsLHEtoEOSManager.py.

int cmsLHEtoEOSManager.newArt = 0

Definition at line 234 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.oldArt = lastArticle()

Definition at line 240 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.parser = optparse.OptionParser(usage)

Definition at line 115 of file cmsLHEtoEOSManager.py.

cmsLHEtoEOSManager.reallyDoIt = notoptions.dryRun

Definition at line 162 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCheckIntegrityCommand = 'xmllint -noout '

Definition at line 212 of file cmsLHEtoEOSManager.py.

list cmsLHEtoEOSManager.theCheckSumList = []

Definition at line 198 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCommand = defaultEOSmkdirCommand+' '

Definition at line 244 of file cmsLHEtoEOSManager.py.

list cmsLHEtoEOSManager.theCompressedFilesList = []

Definition at line 197 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCompressionCommand = 'xz '

Definition at line 221 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.theList = []

Definition at line 182 of file cmsLHEtoEOSManager.py.

Referenced by MuonAlignment.recursiveList().

string cmsLHEtoEOSManager.uploadPath = ''

Definition at line 235 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.usage = 'cmsLHEtoEOSManager.py <options>'

Definition at line 114 of file cmsLHEtoEOSManager.py.