17 #include <sys/types.h> 27 #include <boost/property_tree/json_parser.hpp> 28 #include <boost/property_tree/ptree.hpp> 29 #include <boost/filesystem.hpp> 30 #include <boost/format.hpp> 39 if (value < 1 && value != -1)
40 throw cms::Exception(
"DQMFileSaver") <<
"Invalid '" << name <<
"' parameter '" << value
41 <<
"'. Must be -1 or >= 1.";
60 wflow.reserve(workflow.size() + 3);
62 while ((pos = wflow.find(
'/', pos)) != std::string::npos)
63 wflow.replace(pos++, 1,
"__");
71 sprintf(suffix,
"R%09d", run);
78 sprintf(suffix,
"R%09d", run);
82 sprintf(rewrite,
"\\1Run %d/\\2/Run summary", run);
84 sprintf(rewrite,
"\\1Run %d/\\2/By Lumi Section %d-%d", run, lumi, lumi);
99 me =
dbe_->
get(
"Info/ProvInfo/runIsComplete");
112 "^(Reference/)?([^/]+)",
127 msg <<
"DQMFileSaver: storing EventInfo folders for Run: " << run <<
", Lumi Section: " << lumi <<
", Subsystems: ";
129 for (
size_t i = 0,
e = systems.size();
i !=
e; ++
i) {
130 if (systems[
i] !=
"Reference") {
132 msg << systems[
i] <<
" ";
135 systems[
i] +
"/EventInfo",
136 "^(Reference/)?([^/]+)",
166 store->
save(filename, directory, rxpat, rewrite, enableMultiThread ? run : 0, 0, saveref, saveRefQMin);
168 store->
savePB(filename, filterName, enableMultiThread ? run : 0);
182 "^(Reference/)?([^/]+)",
193 for (
size_t i = 0,
e = systems.size();
i !=
e; ++
i) {
194 if (systems[
i] !=
"Reference") {
202 "^(Reference/)?([^/]+)",
214 for (
size_t i = 0,
e = systems.size();
i !=
e; ++
i)
215 if (systems[
i] !=
"Reference") {
217 std::vector<MonitorElement *> pNamesVector =
219 if (!pNamesVector.empty()) {
225 "^(Reference/)?([^/]+)",
231 pNamesVector.clear();
237 for (
size_t i = 0,
e = systems.size();
i !=
e; ++
i)
238 if (systems[
i] !=
"Reference")
244 "^(Reference/)?([^/]+)",
266 namespace bpt = boost::property_tree;
278 boost::filesystem::create_directories(runDir);
280 jsonFilePathName = baseName +
".jsn";
281 openJsonFilePathName = jsonFilePathName +
".open";
284 openHistoFilePathName = histoFilePathName +
".open";
289 if (fileFormat ==
ROOT) {
292 }
else if (fileFormat ==
PB) {
293 openHistoFilePathName =
300 if (fileFormat ==
ROOT) {
307 "^(Reference/)?([^/]+)",
314 }
else if (fileFormat ==
PB) {
318 throw cms::Exception(
"DQMFileSaver") <<
"Internal error, can save files" 319 <<
" only in ROOT or ProtocolBuffer format.";
322 rename(openHistoFilePathName.c_str(), histoFilePathName.c_str());
327 write_json(openJsonFilePathName, pt);
328 rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str());
335 std::map<std::string, std::string>
info;
336 info[
"Source"] =
"DQMStore";
337 info[
"FileClass"] =
"DQM";
372 if (convention ==
"Offline")
374 else if (convention ==
"Online")
376 else if (convention ==
"FilterUnit")
379 throw cms::Exception(
"DQMFileSaver") <<
"Invalid 'convention' parameter '" << convention <<
"'." 380 <<
" Expected one of 'Online' or 'Offline' or 'FilterUnit'.";
388 workflow_.find_first_not_of(
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" 389 "abcdefghijklmnopqrstuvwxyz" 391 "-_/") != std::string::npos)
392 throw cms::Exception(
"DQMFileSaver") <<
"Invalid 'workflow' parameter '" <<
workflow_ <<
"'. Expected '/A/B/C'.";
395 <<
"The 'workflow' parameter must be empty in 'Online' and 'FilterUnit' conventions.";
403 if (fileFormat ==
"ROOT")
405 else if (fileFormat ==
"PB")
408 throw cms::Exception(
"DQMFileSaver") <<
"Invalid 'fileFormat' parameter '" << fileFormat <<
"'." 409 <<
" Expected one of 'ROOT' or 'PB'.";
419 <<
"'. Expected 'DQM', 'HLTDQM' or 'Playback'.";
433 if (refsave ==
"default")
435 else if (refsave ==
"skip") {
438 }
else if (refsave ==
"all") {
441 }
else if (refsave ==
"qtests") {
445 throw cms::Exception(
"DQMFileSaver") <<
"Invalid 'referenceHandling' parameter '" << refsave
446 <<
"'. Expected 'default', 'skip', 'all' or 'qtests'.";
454 if (dirName_.empty() ||
stat(dirName_.c_str(), &
s) == -1)
455 throw cms::Exception(
"DQMFileSaver") <<
"Invalid 'dirName' parameter '" << dirName_ <<
"'.";
482 sprintf(version,
"_V%04d_",
int(
version_));
485 }
else if (fakeFilterUnitMode_) {
487 <<
"Fake FU mode, files are saved under <dirname>/runXXXXXX/runXXXXXX_lsXXXX_<stream_Label>.pb.\n";
490 edm::LogInfo(
"DQMFileSaver") <<
"The base dir provided in the configuration '" << dirName_ <<
"'\n" 491 <<
" will not be considered: for FU, the DAQ2 services will handle directories\n";
496 if (!(
fms_ && daqDirector))
497 throw cms::Exception(
"DQMFileSaver") <<
"Internal error, cannot initialize DAQ services.";
501 edm::LogInfo(
"DQMFileSaver") <<
"DQM file saving settings:\n" 505 <<
" saving every " <<
saveByRun_ <<
" run(s)\n" 506 <<
" saving at job end: " << (
saveAtJobEnd_ ?
"yes" :
"no") <<
"\n";
532 std::ofstream
file(initFileName);
553 int irun = iLS.
id().
run();
556 throw cms::Exception(
"DQMFileSaver") <<
"Internal error, can save files at end of lumi block" 557 <<
" only in Online, FilterUnit or Offline mode.";
563 sprintf(suffix,
"_R%09d_L%06d", irun, ilumi);
564 sprintf(rewrite,
"\\1Run %d/\\2/By Lumi Section %d-%d", irun, ilumi -
nlumi_, ilumi);
570 throw cms::Exception(
"DQMFileSaver") <<
"Internal error, can save files" 571 <<
" only in ROOT or ProtocolBuffer format.";
584 sprintf(rewrite,
"\\1Run %d/\\2/By Lumi Section %d-%d", irun, ilumi, ilumi);
593 throw cms::Exception(
"DQMFileSaver") <<
"Internal error, can save files" 594 <<
" only in ROOT format.";
603 int irun = iRun.
id().
run();
608 sprintf(suffix,
"_R%09d", irun);
610 sprintf(rewrite,
"\\1Run %d/\\2/Run summary", irun);
616 throw cms::Exception(
"DQMFileSaver") <<
"Internal error, can save files" 617 <<
" only in ROOT or ProtocolBuffer format.";
623 throw cms::Exception(
"DQMFileSaver") <<
"Internal error. Can only save files in endRun()" 624 <<
" in Online and Offline modes.";
629 edm::LogInfo(
"DQMFileSaver") <<
"Producing fake EoR file.\n";
633 std::string openJsonFilePathName = jsonFilePathName +
".open";
635 boost::filesystem::create_directories(runDir);
637 using namespace boost::property_tree;
641 ptree child1, child2, child3;
647 data.push_back(std::make_pair(
"", child1));
648 data.push_back(std::make_pair(
"", child2));
649 data.push_back(std::make_pair(
"", child3));
651 pt.add_child(
"data", data);
652 pt.put(
"definition",
"/non-existant/");
653 pt.put(
"source",
"--hostname--");
655 std::ofstream
file(jsonFilePathName);
656 write_json(file, pt,
true);
659 rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str());
670 throw cms::Exception(
"DQMFileSaver") <<
"Internal error. Can only save files at the end of the" 671 <<
" job in Offline mode.";
MonitorElement * bookString(char_string const &name, char_string const &value)
Book string.
LuminosityBlockID id() const
T getUntrackedParameter(std::string const &, T const &) const
DQMFileSaver(const edm::ParameterSet &ps)
static void doSaveForOnline(DQMFileSaver::DQMStore *store, int run, bool enableMultiThread, const std::string &filename, const std::string &directory, const std::string &rxpat, const std::string &rewrite, DQMFileSaver::DQMStore::SaveReferenceTag saveref, int saveRefQMin, const std::string &filterName, DQMFileSaver::FileFormat fileFormat)
unsigned int getEventsProcessedForLumi(unsigned int lumi, bool *abortFlag=0)
std::shared_ptr< saverDetails::NoCache > globalBeginRun(const edm::Run &, const edm::EventSetup &) const override
void saveForOffline(const std::string &workflow, int run, int lumi) const
void savePB(std::string const &filename, std::string const &path="", uint32_t run=0, uint32_t lumi=0)
void globalEndRun(const edm::Run &, const edm::EventSetup &) const override
static std::string dataFileExtension(DQMFileSaver::FileFormat fileFormat)
std::string getInitFilePath(std::string const &stream) const
evf::FastMonitoringService * fms_
bool shouldWriteFiles(unsigned int lumi, unsigned int *proc=0)
std::atomic< int > nlumi_
void cd()
go to top directory (ie. root)
std::string fileBaseName_
void deleteUnusedLumiHistograms(uint32_t run, uint32_t lumi)
MonitorElement * get(std::string const &path) const
get ME from full pathname (e.g. "my/long/dir/my_histo")
std::string transferDestination_
void saveJobReport(const std::string &filename) const
static void getAnInt(const edm::ParameterSet &ps, int &value, const std::string &name)
static std::string onlineOfflineFileName(const std::string &fileBaseName, const std::string &suffix, const std::string &workflow, const std::string &child, DQMFileSaver::FileFormat fileFormat)
void saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const
std::shared_ptr< saverDetails::NoCache > globalBeginLuminosityBlock(const edm::LuminosityBlock &, const edm::EventSetup &) const override
std::string stream_label_
static const std::string streamPrefix_
std::string getReleaseVersion()
void saveForOnlinePB(int run, const std::string &suffix) const
std::atomic< int > fileUpdate_
LuminosityBlockNumber_t luminosityBlock() const
static boost::property_tree::ptree fillJson(int run, int lumi, const std::string &dataFilePathName, const std::string &transferDestinationStr, const std::string &mergeTypeStr, evf::FastMonitoringService *fms)
void setCurrentFolder(std::string const &fullpath)
std::vector< std::string > getSubdirs() const
void globalEndLuminosityBlock(const edm::LuminosityBlock &, const edm::EventSetup &) const override
void saveForOfflinePB(const std::string &workflow, int run) const
char data[epos_bytes_allocation]
void reportAnalysisFile(std::string const &fileName, std::map< std::string, std::string > const &fileData)
MonitorElement * bookFloat(char_string const &name)
Book float.
std::vector< MonitorElement * > getMatchingContents(std::string const &pattern, lat::Regexp::Syntax syntaxType=lat::Regexp::Wildcard) const
static const int STATUS_OK
void analyze(edm::StreamID, const edm::Event &e, const edm::EventSetup &) const override
void save(std::string const &filename, std::string const &path="", std::string const &pattern="", std::string const &rewrite="", uint32_t run=0, uint32_t lumi=0, SaveReferenceTag ref=SaveWithReference, int minStatus=dqm::qstatus::STATUS_OK, std::string const &fileupdate="RECREATE")
static const std::string streamSuffix_
boost::property_tree::ptree fillJson(int run, int lumi, const std::string &dataFilePathName, const std::string &transferDestinationStr, const std::string &mergeTypeStr, evf::FastMonitoringService *fms)
void saveForFilterUnit(const std::string &rewrite, int run, int lumi, const FileFormat fileFormat) const