CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
FastMonitor.cc
Go to the documentation of this file.
1 /*
2  * FastMonitor.cc
3  *
4  * Created on: Nov 27, 2012
5  * Author: aspataru
6  */
7 
12 
13 #include <fstream>
14 #include <iostream>
15 #include <sstream>
16 #include <cassert>
17 #include <sys/types.h>
18 #include <unistd.h>
19 
20 using namespace jsoncollector;
21 
23  std::string const& defPath, std::string const defGroup, bool strictChecking, bool useSource, bool useDefinition)
24  : defPath_(defPath),
25  strictChecking_(strictChecking),
26  useSource_(useSource),
27  useDefinition_(useDefinition),
28  nStreams_(1),
29  deleteDef_(true) {
30  //get host and PID info
31  if (useSource)
33 
34  //load definition file
35  dpd_ = new DataPointDefinition();
37 }
38 
39 FastMonitor::FastMonitor(DataPointDefinition* dpd, bool strictChecking, bool useSource, bool useDefinition)
40  : strictChecking_(strictChecking), useSource_(useSource), useDefinition_(useDefinition), nStreams_(1), dpd_(dpd) {
41  //get host and PID info
42  if (useSource)
44 }
45 
47  for (auto dp : dataPoints_)
48  delete dp;
49  if (deleteDef_)
50  delete dpd_;
51  if (deleteDefFast_)
52  delete dpdFast_;
53 }
54 
55 void FastMonitor::addFastPathDefinition(std::string const& defPathFast, std::string const defGroupFast, bool strict) {
56  haveFastPath_ = true;
57  defPathFast_ = defPathFast;
60  fastPathStrictChecking_ = strict;
61  deleteDefFast_ = true;
62 }
63 
64 //per-process variables
66  bool NAifZeroUpdates,
67  unsigned int* nBins) {
69  dp->trackMonitorable(newMonitorable, NAifZeroUpdates);
70  dp->setNBins(nBins);
71  dataPoints_.push_back(dp);
72  dpNameMap_[newMonitorable->getName()] = dataPoints_.size() - 1;
73 
74  //checks if the same name is registered twice
75  assert(uids_.insert(newMonitorable->getName()).second);
76 }
77 
78 //fast path: no merge operation is performed
81  dp->trackMonitorable(newMonitorable, false);
82  dataPointsFastOnly_.push_back(dp);
83 }
84 
85 //per-stream variables
87  std::vector<unsigned int>* inputs,
88  bool NAifZeroUpdates,
89  unsigned int* nBins) {
91  dp->trackVectorUInt(name, inputs, NAifZeroUpdates);
92  dp->setNBins(nBins);
93  dataPoints_.push_back(dp);
94  dpNameMap_[name] = dataPoints_.size() - 1;
95  assert(uids_.insert(name).second);
96 }
97 
98 //atomic variables with guaranteed updates at the time of reading
100  std::vector<AtomicMonUInt*>* inputs,
101  bool NAifZeroUpdates,
102  unsigned int* nBins) {
103  std::string definitionToPass;
104  if (useDefinition_)
105  definitionToPass = defPath_;
106  DataPoint* dp = new DataPoint(definitionToPass, sourceInfo_);
107  dp->trackVectorUIntAtomic(name, inputs, NAifZeroUpdates);
108  dp->setNBins(nBins);
109  dataPoints_.push_back(dp);
110  dpNameMap_[name] = dataPoints_.size() - 1;
111  assert(uids_.insert(name).second);
112 }
113 
114 void FastMonitor::commit(std::vector<unsigned int>* streamLumisPtr) {
115  std::vector<std::string> const& jsonNames = dpd_->getNames();
116  regDpCount_ = dataPoints_.size();
117  if (strictChecking_)
118  assert(jsonNames.size() == regDpCount_);
119 
120  std::map<unsigned int, bool> hasJson;
121  for (unsigned int i = 0; i < jsonNames.size(); i++) {
122  bool notFoundVar = true;
123  for (unsigned int j = 0; j < regDpCount_; j++) {
124  if (dataPoints_[j]->getName() == jsonNames[i]) {
125  dataPoints_[j]->setOperation(dpd_->getOperationFor(i));
126  jsonDpIndex_.push_back(j);
127  hasJson[j] = true;
128  notFoundVar = false;
129  break;
130  }
131  }
132  if (notFoundVar) {
134  //push dummy DP if not registered by the service so that we output required JSON/CSV
135  DataPoint* dummyDp = new DataPoint(sourceInfo_, defPath_);
136  dummyDp->trackDummy(jsonNames[i], true);
137  dataPoints_.push_back(dummyDp);
138  jsonDpIndex_.push_back(dataPoints_.size() - 1);
139  }
140  }
141  for (unsigned int i = 0; i < regDpCount_; i++) {
142  dataPoints_[i]->setStreamLumiPtr(streamLumisPtr);
143  }
144 
145  //fast path:
146  if (haveFastPath_) {
147  std::vector<std::string> const& fjsonNames = dpdFast_->getNames();
149  assert(!(fastPathStrictChecking_ && fjsonNames.size() == fregDpCount_));
150  std::map<unsigned int, bool> fhasJson;
151  for (unsigned int i = 0; i < fjsonNames.size(); i++) {
152  bool notFoundVar = true;
153  for (unsigned int j = 0; j < fregDpCount_; j++) {
154  if (dataPointsFastOnly_[j]->getName() == fjsonNames[i]) {
156  fhasJson[j] = true;
157  notFoundVar = false;
158  break;
159  }
160  }
161  if (notFoundVar) {
162  //try to find variable among slow variables
163 
164  bool notFoundVarSlow = true;
165  for (unsigned int j = 0; j < regDpCount_; j++) {
166  if (dataPoints_[j]->getName() == fjsonNames[i]) {
167  jsonDpIndexFast_.push_back(dataPoints_[j]);
168  //fhasJson[j]=true;
169  notFoundVarSlow = false;
170  break;
171  }
172  }
173 
174  assert(!(fastPathStrictChecking_ && !notFoundVarSlow));
175  //push dummy DP if not registered by the service so that we output required JSON/CSV
176  if (notFoundVarSlow) {
177  DataPoint* dummyDp = new DataPoint(sourceInfo_, defPathFast_);
178  dummyDp->trackDummy(fjsonNames[i], true);
179  dataPointsFastOnly_.push_back(dummyDp);
180  jsonDpIndexFast_.push_back(dummyDp);
181  }
182  }
183  }
184  }
185 }
186 
187 //update everything
188 void FastMonitor::snap(unsigned int ls) {
189  recentSnaps_++;
191  for (unsigned int i = 0; i < regDpCount_; i++) {
192  dataPoints_[i]->snap(ls);
193  }
194 }
195 
196 //update for global variables as most of them are correct only at global EOL
197 void FastMonitor::snapGlobal(unsigned int ls) {
198  recentSnaps_++;
199  for (unsigned int i = 0; i < regDpCount_; i++) {
200  dataPoints_[i]->snapGlobal(ls);
201  }
202 }
203 
204 //update atomic per-stream vars(e.g. event counters) not updating time-based measurements (mini/microstate)
205 void FastMonitor::snapStreamAtomic(unsigned int ls, unsigned int streamID) {
206  recentSnaps_++;
207  for (unsigned int i = 0; i < regDpCount_; i++) {
208  dataPoints_[i]->snapStreamAtomic(ls, streamID);
209  }
210 }
211 
213  //output what was specified in JSON in the same order (including dummies)
214  unsigned int monSize = jsonDpIndexFast_.size();
215  std::stringstream ss;
216  if (monSize) {
217  for (unsigned int j = 0; j < monSize; j++) {
218  ss << jsonDpIndexFast_[j]->fastOutCSV(sid);
219  if (j < monSize - 1)
220  ss << ",";
221  }
222  }
223  return ss.str();
224 }
225 
226 void FastMonitor::outputCSV(std::string const& path, std::string const& csvString) {
227  std::ofstream outputFile;
229  outputFile << defPathFast_ << std::endl;
230  outputFile << csvString << std::endl;
231  outputFile.close();
232 }
233 
234 //get one variable (caller must delete it later)
236  auto it = dpNameMap_.find(name);
237  assert(it != dpNameMap_.end());
238  return dataPoints_[it->second]->mergeAndRetrieveValue(forLumi);
239 }
240 
241 bool FastMonitor::outputFullJSONs(std::string const& pathstem, std::string const& ext, unsigned int lumi, bool output) {
242  LogDebug("FastMonitor") << "SNAP updates -: " << recentSnaps_ << " (by timer: " << recentSnapsTimer_
243  << ") in lumisection ";
244 
246  for (unsigned int i = 0; i < nStreams_; i++) {
247  //merge even if no output
248  Json::Value serializeRoot;
249  for (unsigned int j = 0; j < jsonDpIndex_.size(); j++) {
250  dataPoints_[jsonDpIndex_[j]]->mergeAndSerialize(serializeRoot, lumi, true, i);
251  }
252  if (!output)
253  continue;
254  //get extension
255  std::stringstream tidext;
256  tidext << "_tid" << i;
257  std::string path = pathstem + tidext.str() + ext;
258 
260  std::string&& result = writer.write(serializeRoot);
262  }
263  return output;
264 }
265 
266 bool FastMonitor::outputFullJSON(std::string const& path, unsigned int lumi, bool output) {
267  LogDebug("FastMonitor") << "SNAP updates -: " << recentSnaps_ << " (by timer: " << recentSnapsTimer_
268  << ") in lumisection ";
269 
271  Json::Value serializeRoot;
272  for (unsigned int j = 0; j < jsonDpIndex_.size(); j++) {
273  dataPoints_[jsonDpIndex_[j]]->mergeAndSerialize(serializeRoot, lumi, j == 0, -1);
274  }
275  if (!output)
276  return false;
277 
279  std::string&& result = writer.write(serializeRoot);
281  return true;
282 }
283 
284 void FastMonitor::discardCollected(unsigned int forLumi) {
285  for (auto dp : dataPoints_)
286  dp->discardCollected(forLumi);
287 }
288 
290  std::stringstream hpid;
291  int pid = (int)getpid();
292  char hostname[128];
293  gethostname(hostname, sizeof hostname);
294  hpid << hostname << "_" << pid;
295  sHPid = hpid.str();
296 }
eostools.ls
def ls(path, rec=False)
Definition: eostools.py:349
ext
Definition: memstream.h:15
mps_fire.i
i
Definition: mps_fire.py:428
jsoncollector::FastMonitor::uids_
std::unordered_set< std::string > uids_
Definition: FastMonitor.h:124
MessageLogger.h
jsoncollector::FastMonitor::recentSnapsTimer_
unsigned int recentSnapsTimer_
Definition: FastMonitor.h:120
jsoncollector::DataPointDefinition::getNames
std::vector< std::string > const & getNames()
Definition: DataPointDefinition.h:44
jsoncollector::FastMonitor::dataPoints_
std::vector< DataPoint * > dataPoints_
Definition: FastMonitor.h:112
convertSQLitetoXML_cfg.output
output
Definition: convertSQLitetoXML_cfg.py:72
jsoncollector::FastMonitor::commit
void commit(std::vector< unsigned int > *streamLumisPtr)
Definition: FastMonitor.cc:114
jsoncollector::FastMonitor::outputFullJSON
bool outputFullJSON(std::string const &path, unsigned int lumi, bool output=true)
Definition: FastMonitor.cc:266
jsoncollector::FastMonitor::deleteDefFast_
bool deleteDefFast_
Definition: FastMonitor.h:110
jsoncollector::FastMonitor::getMergedIntJForLumi
JsonMonitorable * getMergedIntJForLumi(std::string const &name, unsigned int forLumi)
Definition: FastMonitor.cc:235
cms::cuda::assert
assert(be >=bs)
edm::second
U second(std::pair< T, U > const &p)
Definition: ParameterSet.cc:222
cscNeutronWriter_cfi.writer
writer
Definition: cscNeutronWriter_cfi.py:6
jsoncollector::FastMonitor::recentSnaps_
unsigned int recentSnaps_
Definition: FastMonitor.h:119
jsoncollector::FastMonitor::sourceInfo_
std::string sourceInfo_
Definition: FastMonitor.h:106
jsoncollector::FastMonitor::getHostAndPID
void getHostAndPID(std::string &sHPid)
Definition: FastMonitor.cc:289
jsoncollector::FastMonitor::jsonDpIndexFast_
std::vector< DataPoint * > jsonDpIndexFast_
Definition: FastMonitor.h:115
FileIO.h
jsoncollector::FastMonitor::registerGlobalMonitorable
void registerGlobalMonitorable(JsonMonitorable *newMonitorable, bool NAifZeroUpdates, unsigned int *nBins=nullptr)
Definition: FastMonitor.cc:65
jsoncollector::FastMonitor::dpdFast_
DataPointDefinition * dpdFast_
Definition: FastMonitor.h:108
jsoncollector::JsonMonitorable::getName
virtual std::string & getName()
Definition: JsonMonitorable.h:40
contentValuesCheck.ss
ss
Definition: contentValuesCheck.py:33
jsoncollector::FastMonitor::fregDpCount_
unsigned int fregDpCount_
Definition: FastMonitor.h:122
jsoncollector::FastMonitor::registerStreamMonitorableUIntVec
void registerStreamMonitorableUIntVec(std::string const &name, std::vector< unsigned int > *inputs, bool NAifZeroUpdates, unsigned int *nBins=nullptr)
Definition: FastMonitor.cc:86
download_sqlite_cfg.outputFile
outputFile
Definition: download_sqlite_cfg.py:5
jsoncollector::DataPointDefinition::getDataPointDefinitionFor
static bool getDataPointDefinitionFor(std::string &defFilePath, DataPointDefinition *dpd, const std::string *defaultGroup=nullptr)
Definition: DataPointDefinition.cc:32
jsoncollector::FastMonitor::nStreams_
unsigned int nStreams_
Definition: FastMonitor.h:104
jsoncollector::DataPointDefinition::getOperationFor
OperationType getOperationFor(unsigned int index)
Definition: DataPointDefinition.cc:78
jsoncollector::FastMonitor::addFastPathDefinition
void addFastPathDefinition(std::string const &defPathFast, std::string const defGroupFast, bool strict)
Definition: FastMonitor.cc:55
Calorimetry_cff.dp
dp
Definition: Calorimetry_cff.py:158
jsoncollector::FastMonitor::defPath_
std::string defPath_
Definition: FastMonitor.h:96
jsoncollector::FastMonitor::outputCSV
void outputCSV(std::string const &path, std::string const &csvString)
Definition: FastMonitor.cc:226
getName
TString getName(TString structure, int layer, TString geometry)
Definition: DMRtrends.cc:235
jsoncollector::FastMonitor::deleteDef_
bool deleteDef_
Definition: FastMonitor.h:109
seedmultiplicitymonitor_newtracking_cfi.nBins
nBins
Definition: seedmultiplicitymonitor_newtracking_cfi.py:8
Json::StyledWriter
Writes a Value in JSON format in a human friendly way.
Definition: writer.h:63
jsoncollector::FastMonitor::haveFastPath_
bool haveFastPath_
Definition: FastMonitor.h:102
AlCaHLTBitMon_QueryRunRegistry.string
string
Definition: AlCaHLTBitMon_QueryRunRegistry.py:256
funct::true
true
Definition: Factorize.h:173
jsoncollector::FastMonitor::snapGlobal
void snapGlobal(unsigned int ls)
Definition: FastMonitor.cc:197
jsoncollector::FastMonitor::outputFullJSONs
bool outputFullJSONs(std::string const &pathstem, std::string const &ext, unsigned int lumi, bool output=true)
Definition: FastMonitor.cc:241
JsonSerializable.h
LogDebug
#define LogDebug(id)
Definition: MessageLogger.h:233
jsoncollector::FastMonitor::fastPathStrictChecking_
bool fastPathStrictChecking_
Definition: FastMonitor.h:99
jsoncollector::FastMonitor::regDpCount_
unsigned int regDpCount_
Definition: FastMonitor.h:121
createfilelist.int
int
Definition: createfilelist.py:10
jsoncollector::FastMonitor::registerStreamMonitorableUIntVecAtomic
void registerStreamMonitorableUIntVecAtomic(std::string const &name, std::vector< AtomicMonUInt * > *inputs, bool NAifZeroUpdates, unsigned int *nBins=nullptr)
Definition: FastMonitor.cc:99
jsoncollector::FileIO::writeStringToFile
static void writeStringToFile(std::string const &filename, std::string &content)
Definition: FileIO.cc:21
PixelMapPlotter.inputs
inputs
Definition: PixelMapPlotter.py:490
jsoncollector::FastMonitor::~FastMonitor
virtual ~FastMonitor()
Definition: FastMonitor.cc:46
jsoncollector::DataPoint
Definition: DataPoint.h:36
jsoncollector::FastMonitor::registerFastGlobalMonitorable
void registerFastGlobalMonitorable(JsonMonitorable *newMonitorable)
Definition: FastMonitor.cc:79
jsoncollector::FastMonitor::useDefinition_
bool useDefinition_
Definition: FastMonitor.h:101
jsoncollector::FastMonitor::dpd_
DataPointDefinition * dpd_
Definition: FastMonitor.h:107
jsoncollector::DataPoint::trackDummy
void trackDummy(std::string const &name, bool setNAifZeroUpdates)
Definition: DataPoint.h:80
jsoncollector::JsonMonitorable
Definition: JsonMonitorable.h:24
jsoncollector
Definition: DataPoint.h:26
jsoncollector::DataPointDefinition
Definition: DataPointDefinition.h:20
jsoncollector::FastMonitor::snapStreamAtomic
void snapStreamAtomic(unsigned int ls, unsigned int streamID)
Definition: FastMonitor.cc:205
Skims_PA_cff.name
name
Definition: Skims_PA_cff.py:17
jsoncollector::FastMonitor::dpNameMap_
std::map< std::string, unsigned int > dpNameMap_
Definition: FastMonitor.h:117
jsoncollector::FastMonitor::snap
void snap(unsigned int ls)
Definition: FastMonitor.cc:188
MillePedeFileConverter_cfg.out
out
Definition: MillePedeFileConverter_cfg.py:31
jsoncollector::FastMonitor::strictChecking_
bool strictChecking_
Definition: FastMonitor.h:98
mps_fire.result
result
Definition: mps_fire.py:311
castor_dqm_sourceclient_file_cfg.path
path
Definition: castor_dqm_sourceclient_file_cfg.py:37
jsoncollector::FastMonitor::FastMonitor
FastMonitor(std::string const &defPath, std::string const defGroup, bool strictChecking, bool useSource=true, bool useDefinition=true)
Definition: FastMonitor.cc:22
jsoncollector::FastMonitor::getCSVString
std::string getCSVString(int sid=-1)
Definition: FastMonitor.cc:212
jsoncollector::FastMonitor::jsonDpIndex_
std::vector< unsigned int > jsonDpIndex_
Definition: FastMonitor.h:114
dqmiolumiharvest.j
j
Definition: dqmiolumiharvest.py:66
jsoncollector::FastMonitor::discardCollected
void discardCollected(unsigned int forLumi)
Definition: FastMonitor.cc:284
lumi
Definition: LumiSectionData.h:20
pileupReCalc_HLTpaths.trunc
trunc
Definition: pileupReCalc_HLTpaths.py:144
jsoncollector::FastMonitor::defPathFast_
std::string defPathFast_
Definition: FastMonitor.h:97
FastMonitor.h
Json::Value
Represents a JSON value.
Definition: value.h:99
jsoncollector::FastMonitor::dataPointsFastOnly_
std::vector< DataPoint * > dataPointsFastOnly_
Definition: FastMonitor.h:113