1 // ANNA - Anna is Not Nothingness Anymore
3 // (c) Copyright 2005-2014 Eduardo Ramos Testillano & Francisco Ruiz Rayo
5 // http://redmine.teslayout.com/projects/anna-suite
7 // Redistribution and use in source and binary forms, with or without
8 // modification, are permitted provided that the following conditions
11 // * Redistributions of source code must retain the above copyright
12 // notice, this list of conditions and the following disclaimer.
13 // * Redistributions in binary form must reproduce the above
14 // copyright notice, this list of conditions and the following disclaimer
15 // in the documentation and/or other materials provided with the
17 // * Neither the name of the copyright holder nor the names of its
18 // contributors may be used to endorse or promote products derived from
19 // this software without specific prior written permission.
21 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 // Authors: eduardo.ramos.testillano@gmail.com
34 // cisco.tierra@gmail.com
37 //------------------------------------------------------------------------------
38 //--------------------------------------------------------------------- #include
39 //------------------------------------------------------------------------------
43 #include <anna/statistics/Accumulator.hpp>
44 #include <anna/statistics/Engine.hpp>
46 #include <anna/time/Date.hpp>
53 #include <anna/xml/xml.hpp>
54 #include <anna/core/tracing/Logger.hpp>
55 #include <anna/core/tracing/TraceMethod.hpp>
59 using namespace anna::statistics;
60 using namespace anna::time;
63 //******************************************************************************
64 //------------------------------------------------------------------------------
65 //--------------------------------------------------- Accumulator::Accumulator()
66 //------------------------------------------------------------------------------
68 // Default Constructor
69 Accumulator::Accumulator() {
74 Accumulator::~Accumulator() {
75 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "Destructor", ANNA_FILE_LOCATION));
83 //------------------------------------------------------------------------------
84 //---------------------------------------------------- Accumulator::initialize()
85 //------------------------------------------------------------------------------
86 void Accumulator::initialize(const int & conceptId) throw() {
87 _concept_data_t conceptData;
89 a_concept_data_map[conceptId] = conceptData;
93 //------------------------------------------------------------------------------
94 //---------------------------------------------------- Accumulator::getConcept()
95 //------------------------------------------------------------------------------
96 _concept_data_t * Accumulator::getConcept(const int & conceptId) const throw(anna::RuntimeException) {
97 _concept_data_map_iter it = a_concept_data_map.find(conceptId);
99 if(it == a_concept_data_map.end()) { // not found
100 // Check if concept id is registered:
101 Engine& engine = Engine::instantiate();
102 std::string conceptDescription, conceptUnitDescription;
105 if(!engine.getConcept(conceptId, conceptDescription, conceptUnitDescription, integerNature)) {
106 std::string msg = "Can't find at engine nothing about concept id = ";
107 msg += anna::functions::asString(conceptId);
108 throw anna::RuntimeException(msg, ANNA_FILE_LOCATION);
111 Accumulator * nc_this = const_cast<Accumulator *>(this);
112 nc_this->initialize(conceptId);
113 // (la otra posibilidad era hacer mutable el mapa e inicializar poniendo el contenido de 'initialize()')
114 it = a_concept_data_map.find(conceptId);
117 return ((_concept_data_t *) & ((*it).second));
121 //------------------------------------------------------------------------------
122 //--------------------------------------------------- Accumulator::floatFormat()
123 //------------------------------------------------------------------------------
124 std::string Accumulator::floatFormat(const int & numberOfDecimals) const throw() {
125 std::string result = "%.";
126 result += anna::functions::asString(numberOfDecimals);
135 //------------------------------------------------------------------------------
136 //------------------------------------------ Accumulator::getStandardDeviation()
137 //------------------------------------------------------------------------------
138 double Accumulator::getStandardDeviation(const _concept_data_t * conceptData) const throw(anna::RuntimeException) {
139 // SD = sqrt (1/N SUM (xi^2) - X^2) = sqrt (SquareSum / N - Average^2)
140 if(conceptData->Size == 0)
141 throw anna::RuntimeException("Divide by zero: sample size = 0 for Standard Deviation !!", ANNA_FILE_LOCATION);
143 return (::sqrt((conceptData->SquareSum / conceptData->Size) - (::pow(conceptData->Average, (double)2))));
147 //------------------------------------------------------------------------------
148 //------------------------------------ Accumulator::getBesselStandardDeviation()
149 //------------------------------------------------------------------------------
150 double Accumulator::getBesselStandardDeviation(const _concept_data_t * conceptData) const throw(anna::RuntimeException) {
151 // BSD = sqrt (1/(N-1) SUM (xi^2) - N/(N-1) X^2) = sqrt (SquareSum / (N-1) - N/(N-1) Average^2)
152 if(conceptData->Size == 1)
153 throw anna::RuntimeException("Divide by zero: sample size = 1 for bessel's Standard Deviation !!", ANNA_FILE_LOCATION);
155 double N = conceptData->Size;
156 return (::sqrt((conceptData->SquareSum / (N - 1)) - (N / (N - 1)) * (::pow(conceptData->Average, (double)2))));
164 //------------------------------------------------------------------------------
165 //------------------------------------------------------- Accumulator::process()
166 //------------------------------------------------------------------------------
167 void Accumulator::process(const int & conceptId, const double & value) throw(anna::RuntimeException) {
168 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "process", ANNA_FILE_LOCATION));
169 Engine& engine = Engine::instantiate();
171 if(!engine.enabled()) return;
173 _concept_data_t *ptr_auxConceptData = getConcept(conceptId);
174 anna::Millisecond current = anna::Millisecond::getTime();
175 // Optional sample file dump:
176 Engine::instantiate().logSample(conceptId, current, value);
178 // Statistics calculations
179 if(ptr_auxConceptData->Size == ULLONG_MAX) // Statistics is better during processing until reset
180 ptr_auxConceptData->reset();
182 ptr_auxConceptData->Size ++;
185 if(ptr_auxConceptData->Size == 1) {
186 ptr_auxConceptData->MinimumProcessed = value;
187 ptr_auxConceptData->MaximumProcessed = value;
188 ptr_auxConceptData->MinimumEventTimestampMs = current;
189 ptr_auxConceptData->MaximumEventTimestampMs = current;
191 if(value < ptr_auxConceptData->MinimumProcessed) {
192 ptr_auxConceptData->MinimumProcessed = value;
193 ptr_auxConceptData->MinimumEventTimestampMs = current;
196 if(value > ptr_auxConceptData->MaximumProcessed) {
197 ptr_auxConceptData->MaximumProcessed = value;
198 ptr_auxConceptData->MaximumEventTimestampMs = current;
202 ptr_auxConceptData->Sum += value;
203 ptr_auxConceptData->SquareSum += ::pow(value, (double)2); // (*) standard deviation formula simplification
204 ptr_auxConceptData->Average = ptr_auxConceptData->Sum / ptr_auxConceptData->Size;
205 //ptr_auxConceptData->SquareSumSampleDeviation += ::pow(value - ptr_auxConceptData->Average, (double)2);
206 // Don't use former method to get standard deviation because is not correct: in each processing, average
207 // will be change (is not fixed). There is a simplification for standard deviation, based on square sum (*):
208 // Take X as average for x1, x2, x3, etc., xN
209 // SUM (xi - X)^2 = SUM (xi^2 + X^2 - 2xiX) = SUM (xi^2) + NX^2 - 2X SUM (xi) =
210 // = SUM (xi`2) + NX^2 - 2X(NX) = SUM (xi^2) - NX^2
212 // Then, SD = sqrt (1/N SUM (xi^2) - X^2) = sqrt (SquareSum / N - Average^2)
213 // And corrected SD (Bessel's standard deviation) is:
214 // BSD = sqrt (1/(N-1) SUM (xi^2) - N/(N-1) X^2) = sqrt (SquareSum / (N-1) - N/(N-1) Average^2)
215 // All these values don't need to be calculated now
216 // Worked with pointer
217 //a_concept_data_map[conceptId] = *ptr_auxConceptData;
221 //------------------------------------------------------------------------------
222 //------------------------------------------------------- Accumulator::operator=
223 //------------------------------------------------------------------------------
224 const Accumulator & Accumulator::operator = (const Accumulator & accumulator) {
225 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "operator=", ANNA_FILE_LOCATION));
226 // Avoid self copy: i.e., Accumulator a,b; a=&b; a=b; b=a;
227 if(this == &accumulator) return (*this);
229 a_concept_data_map = accumulator.a_concept_data_map;
235 //------------------------------------------------------------------------------
236 //--------------------------------------------------------- Accumulator::reset()
237 //------------------------------------------------------------------------------
238 void Accumulator::reset(void) throw() {
239 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "reset", ANNA_FILE_LOCATION));
240 _concept_data_map_iter it;
241 _concept_data_map_iter it_min(a_concept_data_map.begin());
242 _concept_data_map_iter it_max(a_concept_data_map.end());
244 for(it = it_min; it != it_max; it++) {
245 //reset ((*it).first);
246 _concept_data_t * ptr_auxConceptData = (_concept_data_t*) & ((*it).second);
247 ptr_auxConceptData->reset();
252 //------------------------------------------------------------------------------
253 //--------------------------------------------------------- Accumulator::reset()
254 //------------------------------------------------------------------------------
255 void Accumulator::reset(const int & conceptId) throw(anna::RuntimeException) {
256 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "reset", ANNA_FILE_LOCATION));
257 _concept_data_t *ptr_auxConceptData = getConcept(conceptId);
258 //if (ptr_auxConceptData == NULL) return; // Not possible: getConcept initializes it if not found.
259 ptr_auxConceptData->reset();
265 //------------------------------------------------------------------------------
266 //---------------------------------------------------- Accumulator::sampleSize()
267 //------------------------------------------------------------------------------
268 unsigned long long int Accumulator::sampleSize(const int & conceptId) const throw(anna::RuntimeException) {
269 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "sampleSize", ANNA_FILE_LOCATION));
270 _concept_data_t *ptr_auxConceptData = getConcept(conceptId);
271 //if (ptr_auxConceptData == NULL) return 0; // Not possible: getConcept initializes it if not found.
272 return (ptr_auxConceptData->Size);
276 //------------------------------------------------------------------------------
277 //------------------------------------------------------ Accumulator::getValue()
278 //------------------------------------------------------------------------------
279 double Accumulator::getValue(const int & conceptId, const Operation::Type & operation) const throw(anna::RuntimeException) {
280 // LOGMETHOD (TraceMethod tttm ("anna::statistics::Accumulator", "getValue", ANNA_FILE_LOCATION));
281 const _concept_data_t *ptr_auxConceptData = getConcept(conceptId);
284 //case Operation::Sum:
285 // return (ptr_auxConceptData->Sum);
286 case Operation::Average:
287 return (ptr_auxConceptData->Average);
288 case Operation::StandardDeviation:
289 return (getStandardDeviation(ptr_auxConceptData));
290 case Operation::BesselStandardDeviation:
291 return (getBesselStandardDeviation(ptr_auxConceptData));
292 case Operation::Minimum:
293 return (ptr_auxConceptData->MinimumProcessed);
294 case Operation::Maximum:
295 return (ptr_auxConceptData->MaximumProcessed);
297 throw anna::RuntimeException("Unrecognized operation", ANNA_FILE_LOCATION);
300 //if (ptr_auxConceptData == NULL) return ((double)0); // Not possible: getConcept initializes it if not found.
304 //------------------------------------------------------------------------------
305 //------------------------------------------------------ Accumulator::asString()
306 //------------------------------------------------------------------------------
307 std::string Accumulator::asString(const int & numberOfDecimals) const throw() {
309 _concept_data_map_iter iter;
310 _concept_data_map_iter iter_min(a_concept_data_map.begin());
311 _concept_data_map_iter iter_max(a_concept_data_map.end());
312 time_t unixTimestamp;
313 time::Date time_now; // set current time on local TZ
314 time::Date time_aux; // for events
315 Engine& engine = Engine::instantiate();
316 std::string fFormat = floatFormat(numberOfDecimals);
317 trace = "\n=====================";
318 trace += "\nStatistic Information";
319 trace += "\n=====================";
320 trace += "\nCurrent Time: ";
321 trace += time_now.asString();
323 if(a_concept_data_map.size() == 0) {
324 trace += "\nNo concept data found (empty map)";
328 trace += "\nNumber of elements (measured concepts) = "; trace += anna::functions::asString(a_concept_data_map.size()); trace += "\n";
330 std::string conceptDescription;
331 std::string conceptUnitDescription;
332 bool integerNatureSample;
334 for(iter = iter_min; iter != iter_max; iter++) {
335 conceptId = (*iter).first;
336 const _concept_data_t * ptrConceptData = &((*iter).second);
337 engine.getConcept(conceptId, conceptDescription, conceptUnitDescription, integerNatureSample); // never launch exception
338 trace += "\n Concept: [";
339 trace += anna::functions::asString(conceptId);
341 trace += conceptDescription; trace += " ("; trace += conceptUnitDescription; trace += ")";
343 if(ptrConceptData->Size != 0) {
344 trace += "\n Sample Size: "; trace += anna::functions::asString((anna::U64)(ptrConceptData->Size));
345 trace += "\n Average: "; trace += anna::functions::asString(fFormat.c_str(), ptrConceptData->Average);
346 trace += "\n Standard Deviation: "; trace += anna::functions::asString(fFormat.c_str(), getStandardDeviation(ptrConceptData));
348 if(ptrConceptData->Size != 1) {
349 trace += "\n Bessel's Standard Deviation: "; trace += anna::functions::asString(fFormat.c_str(), getBesselStandardDeviation(ptrConceptData));
352 trace += "\n Minimum: ";
354 if(integerNatureSample)
355 trace += anna::functions::asString((int)(ptrConceptData->MinimumProcessed));
357 trace += anna::functions::asString(fFormat.c_str(), ptrConceptData->MinimumProcessed);
359 // Minimum Timestamp:
360 unixTimestamp = (ptrConceptData->MinimumEventTimestampMs / 1000); // not neccessary such precision
361 time_aux.storeUnix(unixTimestamp);
362 trace += " ("; trace += time_aux.asString(); trace += ")";
363 trace += "\n Maximum: ";
365 if(integerNatureSample)
366 trace += anna::functions::asString((int)(ptrConceptData->MaximumProcessed));
368 trace += anna::functions::asString(fFormat.c_str(), ptrConceptData->MaximumProcessed);
370 // Maximum Timestamp:
371 unixTimestamp = (ptrConceptData->MaximumEventTimestampMs / 1000); // not neccessary such precision
372 time_aux.storeUnix(unixTimestamp);
373 trace += " ("; trace += time_aux.asString(); trace += ")";
375 unixTimestamp = (ptrConceptData->LastResetEventTimestampMs / 1000); // not neccessary such precision
376 time_aux.storeUnix(unixTimestamp);
377 trace += "\n Last Reset Timestamp: "; trace += time_aux.asString();
379 // Processing Rate: it is not updated at 'process' to avoid 'time_now' use on every call
380 if(time_now != time_aux) {
381 trace += "\n Processing rate: ";
382 int lapse = time_now.getUnixTimestamp() - time_aux.getUnixTimestamp();
383 trace += anna::functions::asString(fFormat.c_str(), ptrConceptData->Size / ((double)lapse));
384 trace += " vps (values per second)";
387 trace += "\n Not enough data to get statistics";
396 //------------------------------------------------------------------------------
397 //--------------------------------------------------------- Accumulator::asXML()
398 //------------------------------------------------------------------------------
399 anna::xml::Node* Accumulator::asXML(anna::xml::Node* parent, const int & numberOfDecimals) const throw() {
400 anna::xml::Node* result = parent->createChild("anna.statistics.Accumulator");
401 _concept_data_map_iter iter;
402 _concept_data_map_iter iter_min(a_concept_data_map.begin());
403 _concept_data_map_iter iter_max(a_concept_data_map.end());
404 time_t unixTimestamp;
405 time::Date time_now; // set current time on local TZ
406 time::Date time_aux; // for events
407 Engine& engine = Engine::instantiate();
408 std::string fFormat = floatFormat(numberOfDecimals);
409 //result->createAttribute("CurrentTime", time_now.asString());
410 anna::xml::Node* currentTime = result->createChild("CurrentTime");
411 time_now.asXML(currentTime);
413 if(a_concept_data_map.size() == 0) {
414 result->createAttribute("ConceptDataMapSize", "0: No concept data found (empty map)");
418 result->createAttribute("ConceptDataMapSize", anna::functions::asString(a_concept_data_map.size()));
420 std::string conceptDescription;
421 std::string conceptUnitDescription;
422 bool integerNatureSample;
425 for(iter = iter_min; iter != iter_max; iter++) {
426 conceptId = (*iter).first;
427 const _concept_data_t * ptrConceptData = &((*iter).second);
428 engine.getConcept(conceptId, conceptDescription, conceptUnitDescription, integerNatureSample); // never launch exception
429 anna::xml::Node* concept = result->createChild("Concept");
430 concept->createAttribute("Id", conceptId);
431 concept->createAttribute("Description", conceptDescription);
432 concept->createAttribute("Unit", conceptUnitDescription);
433 anna::xml::Node* data = concept->createChild("Data");
435 if(ptrConceptData->Size != 0) {
436 data->createAttribute("Size", anna::functions::asString((anna::U64)ptrConceptData->Size));
437 data->createAttribute("Average", anna::functions::asString(fFormat.c_str(), ptrConceptData->Average));
438 data->createAttribute("StandardDeviation", anna::functions::asString(fFormat.c_str(), getStandardDeviation(ptrConceptData)));
440 if(ptrConceptData->Size != 1)
441 data->createAttribute("BesselStandardDeviation", anna::functions::asString(fFormat.c_str(), getBesselStandardDeviation(ptrConceptData)));
444 anna::xml::Node* minimum = data->createChild("Minimum");
446 if(integerNatureSample)
447 minimum->createAttribute("Value", anna::functions::asString((int)(ptrConceptData->MinimumProcessed)));
449 minimum->createAttribute("Value", anna::functions::asString(fFormat.c_str(), ptrConceptData->MinimumProcessed));
451 unixTimestamp = (ptrConceptData->MinimumEventTimestampMs / 1000); // not neccessary such precision
452 time_aux.storeUnix(unixTimestamp);
453 time_aux.asXML(minimum);
455 anna::xml::Node* maximum = data->createChild("Maximum");
457 if(integerNatureSample)
458 maximum->createAttribute("Value", anna::functions::asString((int)(ptrConceptData->MaximumProcessed)));
460 maximum->createAttribute("Value", anna::functions::asString(fFormat.c_str(), ptrConceptData->MaximumProcessed));
462 unixTimestamp = (ptrConceptData->MaximumEventTimestampMs / 1000); // not neccessary such precision
463 time_aux.storeUnix(unixTimestamp);
464 time_aux.asXML(maximum);
466 anna::xml::Node* lastResetEvent = data->createChild("LastResetTimestamp");
467 unixTimestamp = (ptrConceptData->LastResetEventTimestampMs / 1000); // not neccessary such precision
468 time_aux.storeUnix(unixTimestamp);
469 time_aux.asXML(lastResetEvent);
471 // Processing Rate: it is not updated at 'process' to avoid 'time_now' use on every call
472 if(time_now != time_aux) {
473 int lapse = time_now.getUnixTimestamp() - time_aux.getUnixTimestamp();
474 cad_aux = anna::functions::asString(fFormat.c_str(), ptrConceptData->Size / ((double)lapse)); cad_aux += " vps (values per second)";
475 data->createAttribute("ProcessingRate", cad_aux);
478 concept->createAttribute("Data", "Not enough data to get statistics");