From: Eduardo Ramos Testillano Date: Tue, 6 Oct 2015 15:57:37 +0000 (+0200) Subject: Basic stat summary for testcases X-Git-Tag: REFACTORING_TESTING_LIBRARY~100 X-Git-Url: https://git.teslayout.com/public/public/public/?p=anna.git;a=commitdiff_plain;h=13b6f0196c2091f9e88a0778155cdcb733cfdafd Basic stat summary for testcases --- diff --git a/example/diameter/launcher/Launcher.cpp b/example/diameter/launcher/Launcher.cpp index edeaf88..e8ac212 100644 --- a/example/diameter/launcher/Launcher.cpp +++ b/example/diameter/launcher/Launcher.cpp @@ -1354,7 +1354,9 @@ std::string Launcher::help() const throw() { result += "\n"; result += "\n test|clear Clears all the programmed test cases and stop testing (if in progress)."; result += "\n"; - result += "\n test|stat Global statistics."; + result += "\n test|summary Test manager general report (number of test cases, counts by state, global configuration,"; + result += "\n forced in-progress limitation, reports visibility, etc.). Be careful when you have reports"; + result += "\n dumps enabled because all the programmed test cases will be dump and that could be heavy."; result += "\n"; result += "\n"; result += "\nUSING OPERATIONS INTERFACE"; @@ -1747,6 +1749,8 @@ void Launcher::eventOperation(const std::string &operation, std::string &respons // test|interact|amount|id Makes interactive a specific test case id. The amount is the margin of execution steps ... // test|reset|[|id] Reset the test case for id provided, all the tests when missing ... // test|clear Clears all the programmed test cases. + // test|summary Test manager general report (number of test cases, counts by state ... + if(param1 == "ttps") { if (numParams > 2) @@ -1916,6 +1920,10 @@ void Launcher::eventOperation(const std::string &operation, std::string &respons opt_response_content = "there are not programmed test cases to be removed"; } } + else if(param1 == "summary") { + response_content = testManager.asXMLString(); + return; + } else { int id = atoi(param1.c_str()); if(id < 0) @@ -2119,7 +2127,7 @@ throw() { anna::statistics::Engine::instantiate().asXML(result); // Testing: could be heavy if test case reports are enabled - //TestManager::instantiate().asXML(result); + TestManager::instantiate().asXML(result); return result; } diff --git a/example/diameter/launcher/testing/TestCase.cpp b/example/diameter/launcher/testing/TestCase.cpp index e95a670..3d91f32 100644 --- a/example/diameter/launcher/testing/TestCase.cpp +++ b/example/diameter/launcher/testing/TestCase.cpp @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include // Project @@ -30,6 +28,7 @@ #include +/////////////////////////////////////////////////////////////////////////////////////////////////// void TestCase::DebugSummary::addHint(const std::string &hint) throw() { event_t event; event.Timestamp = anna::functions::millisecond(); @@ -53,12 +52,24 @@ anna::xml::Node* TestCase::DebugSummary::asXML(anna::xml::Node* parent) const th return result; }; +/////////////////////////////////////////////////////////////////////////////////////////////////// +TestCase::TestCase(unsigned int id) : + a_id(id), + a_state(State::Initialized), + a_startTime(0), + a_interactiveAmount(-1) { + + /*a_stepsIt = a_steps.end()*/; + TestManager &testManager = TestManager::instantiate(); + testManager.tcsStateStats(State::Initialized, State::Initialized); +} + TestCase::~TestCase() { reset(true); // hard reset - std::map::const_iterator it; - for (it = a_steps.begin(); it != a_steps.end(); it++) delete (it->second); + std::vector::const_iterator it; + for (it = a_steps.begin(); it != a_steps.end(); it++) delete (*it); } const char* TestCase::asText(const State::_v state) @@ -77,9 +88,9 @@ throw() { int steps = a_steps.size(); if (steps != 0) { result->createAttribute("NumberOfTestSteps", steps); - std::map::const_iterator it; + std::vector::const_iterator it; for (it = a_steps.begin(); it != a_steps.end(); it++) { - it->second->asXML(result); + (*it)->asXML(result); } } @@ -98,11 +109,11 @@ std::string TestCase::asXMLString() const throw() { } bool TestCase::hasSameCondition(const TestCondition &condition) const throw() { - std::map::const_iterator it; + std::vector::const_iterator it; TestStepWait *step; for (it = a_steps.begin(); it != a_steps.end(); it++) { - if (it->second->getType() != TestStep::Type::Wait) continue; - step = (TestStepWait *)(it->second); + if ((*it)->getType() != TestStep::Type::Wait) continue; + step = (TestStepWait *)(*it); if (step->getCondition() == condition) return true; } return false; @@ -115,6 +126,11 @@ void TestCase::setState(const State::_v &state) throw() { if (state == previousState) return; a_state = state; TestManager &testManager = TestManager::instantiate(); + + // stats: + testManager.tcsStateStats(previousState, state); + + if (isFinished()) { if (!testManager.getDumpReports()) return; // report file name: cycle-.testcase-.xml @@ -142,14 +158,6 @@ void TestCase::setState(const State::_v &state) throw() { out.close(); } } - - // Count in-progress test cases: - if (inProgress()) { - testManager.setInProgressCountDelta(1); - } - else if (previousState == State::InProgress){ - testManager.setInProgressCountDelta(-1); - } } bool TestCase::done() throw() { @@ -162,7 +170,7 @@ bool TestCase::done() throw() { } bool TestCase::process() throw() { - if (a_steps.size() == 0) { + if (steps() == 0) { LOGWARNING(anna::Logger::warning(anna::functions::asString("Test case %llu is empty, nothing to execute", a_id), ANNA_FILE_LOCATION)); return false; } @@ -183,7 +191,7 @@ bool TestCase::process() throw() { if (done()) return false; bool somethingDone = false; - while (a_stepsIt->second->execute()) { // executes returns 'true' if the next step must be also executed (execute until can't stand no more) + while ((*a_stepsIt)->execute()) { // executes returns 'true' if the next step must be also executed (execute until can't stand no more) nextStep(); // Check end of the test case: if (done()) return false; @@ -200,9 +208,9 @@ bool TestCase::reset(bool hard) throw() { // Clean stage //////////////////////////// // id is kept - std::map::iterator it; + std::vector::iterator it; for (it = a_steps.begin(); it != a_steps.end(); it++) - it->second->reset(); + (*it)->reset(); a_debugSummary.clear(); a_startTime = 0; @@ -367,10 +375,10 @@ void TestCase::addCommand(const std::string &cmd) throw(anna::RuntimeException) TestStepWait *TestCase::searchNextWaitConditionFulfilled(const anna::DataBlock &message, bool waitFromEntity) throw() { TestStepWait *result; - for (std::map::const_iterator it = a_stepsIt /* current */; it != a_steps.end(); it++) { - if (it->second->getType() != TestStep::Type::Wait) continue; - if (it->second->isCompleted()) continue; - result = (TestStepWait*)(it->second); + for (std::vector::const_iterator it = a_stepsIt /* current */; it != a_steps.end(); it++) { + if ((*it)->getType() != TestStep::Type::Wait) continue; + if ((*it)->isCompleted()) continue; + result = (TestStepWait*)(*it); if ((result->getCondition().receivedFromEntity() == waitFromEntity) && (result->fulfilled(message))) return result; } @@ -379,7 +387,7 @@ TestStepWait *TestCase::searchNextWaitConditionFulfilled(const anna::DataBlock & } const TestStep *TestCase::getStep(int stepNumber) const throw() { - std::map::const_iterator it = a_steps.find(stepNumber); - if (it != a_steps.end()) return it->second; - return NULL; + if (stepNumber < 1 || stepNumber > steps()) return NULL; +// return a_steps.at(stepNumber-1); // http://stackoverflow.com/questions/3269809/stdvectorat-vs-operator-surprising-results-5-to-10-times-slower-f + return a_steps[stepNumber-1]; } diff --git a/example/diameter/launcher/testing/TestCase.hpp b/example/diameter/launcher/testing/TestCase.hpp index ab8e4a3..715856a 100644 --- a/example/diameter/launcher/testing/TestCase.hpp +++ b/example/diameter/launcher/testing/TestCase.hpp @@ -57,7 +57,7 @@ public: anna::xml::Node* asXML(anna::xml::Node* parent) const throw(); }; - TestCase(unsigned int id) : a_id(id), a_state(State::Initialized), a_startTime(0), a_interactiveAmount(-1) { /*a_stepsIt = a_steps.end()*/;} + TestCase(unsigned int id); ~TestCase(); struct State { enum _v { Initialized, InProgress, Failed, Success }; }; @@ -109,7 +109,7 @@ public: //helpers int steps() const throw() { return a_steps.size(); } - void addStep(TestStep *step) throw() { a_steps[1+steps()] = step; } + void addStep(TestStep *step) throw() { a_steps.push_back(step); } TestStepWait *searchNextWaitConditionFulfilled(const anna::DataBlock &message, bool waitFromEntity) throw(); // When a message arrives, we identify the test case by mean the Session-Id. Then, from the current step iterator (included), @@ -123,8 +123,8 @@ public: private: // private members: unsigned int a_id; - std::map a_steps; - std::map::const_iterator a_stepsIt; + std::vector a_steps; + std::vector::const_iterator a_stepsIt; std::map a_hopByHops; // for wait-answer State::_v a_state; anna::Millisecond a_startTime; diff --git a/example/diameter/launcher/testing/TestManager.cpp b/example/diameter/launcher/testing/TestManager.cpp index 6dfec1e..ee6be63 100644 --- a/example/diameter/launcher/testing/TestManager.cpp +++ b/example/diameter/launcher/testing/TestManager.cpp @@ -21,7 +21,6 @@ // Process #include -#include #include #include #include @@ -30,8 +29,57 @@ class TestTimer; + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +void TestManager::StatSummary::newTCState(const TestCase::State::_v beginState, const TestCase::State::_v endState) throw() { + + if ((beginState == TestCase::State::Initialized)&&(endState == TestCase::State::Initialized)) { // special case (new test case provisioning) + a_initializedTcs++; + return; + } + + switch (beginState) { + case TestCase::State::Initialized: a_initializedTcs--; break; + case TestCase::State::InProgress: a_inprogressTcs--; break; + case TestCase::State::Failed: a_failedTcs--; break; + case TestCase::State::Success: a_sucessTcs--; break; + default: break; + } + switch (endState) { + case TestCase::State::Initialized: a_initializedTcs++; break; + case TestCase::State::InProgress: a_inprogressTcs++; break; + case TestCase::State::Failed: a_failedTcs++; break; + case TestCase::State::Success: a_sucessTcs++; break; + default: break; + } +} + +void TestManager::StatSummary::clear() throw() { + a_initializedTcs = 0; + a_inprogressTcs = 0; + a_failedTcs = 0; + a_sucessTcs = 0; +} + +anna::xml::Node *TestManager::StatSummary::asXML(anna::xml::Node* parent) const throw() { + anna::xml::Node* result = parent->createChild("StatSummary"); + + anna::xml::Node* tcs = result->createChild("TestCasesCounts"); + tcs->createAttribute("Total", a_initializedTcs + a_inprogressTcs + a_failedTcs + a_sucessTcs); + tcs->createAttribute("Initialized", a_initializedTcs); + tcs->createAttribute("InProgress", a_inprogressTcs); + tcs->createAttribute("Failed", a_failedTcs); + tcs->createAttribute("Success", a_sucessTcs); + + return result; +} +/////////////////////////////////////////////////////////////////////////////////////////////////// + + + TestManager::TestManager() : - anna::timex::TimeEventObserver("TestManager") { + anna::timex::TimeEventObserver("TestManager") { a_timeController = NULL; a_reportsDirectory = "./"; a_dumpReports = false; @@ -39,7 +87,6 @@ TestManager::TestManager() : a_synchronousAmount = 1; a_poolRepeats = 0; // repeat disabled by default a_poolCycle = 1; - a_inProgressCount = 0; a_inProgressLimit = UINT_MAX; // no limit a_clock = NULL; //a_testPool.clear(); @@ -76,9 +123,9 @@ throw(anna::RuntimeException) { result->setTimeout(timeout); LOGDEBUG( - std::string msg("TestManager::createTimer | "); - msg += result->asString(); - anna::Logger::debug(msg, ANNA_FILE_LOCATION); + std::string msg("TestManager::createTimer | "); + msg += result->asString(); + anna::Logger::debug(msg, ANNA_FILE_LOCATION); ); a_timeController->activate(result); @@ -91,9 +138,9 @@ throw() { return; LOGDEBUG( - std::string msg("TestManager::cancel | "); - msg += timer->asString(); - anna::Logger::debug(msg, ANNA_FILE_LOCATION); + std::string msg("TestManager::cancel | "); + msg += timer->asString(); + anna::Logger::debug(msg, ANNA_FILE_LOCATION); ); try { @@ -153,15 +200,15 @@ bool TestManager::configureTTPS(int testTicksPerSecond) throw() { if (a_synchronousAmount > 1) { LOGWARNING( - std::string msg = anna::functions::asString("Desired testing time trigger rate (%d events per second) requires more than one sending per event (%d every %lld milliseconds). Consider launch more instances with lower rate (for example %d ADML processes with %d ttps), or configure %d or more sockets to the remote endpoints to avoid burst sendings", - testTicksPerSecond, - a_synchronousAmount, - admlTimeInterval.getValue(), - a_synchronousAmount, - 1000/admlTimeInterval, - a_synchronousAmount); - - anna::Logger::warning(msg, ANNA_FILE_LOCATION); + std::string msg = anna::functions::asString("Desired testing time trigger rate (%d events per second) requires more than one sending per event (%d every %lld milliseconds). Consider launch more instances with lower rate (for example %d ADML processes with %d ttps), or configure %d or more sockets to the remote endpoints to avoid burst sendings", + testTicksPerSecond, + a_synchronousAmount, + admlTimeInterval.getValue(), + a_synchronousAmount, + 1000/admlTimeInterval, + a_synchronousAmount); + + anna::Logger::warning(msg, ANNA_FILE_LOCATION); ); } @@ -208,6 +255,8 @@ TestCase *TestManager::getTestCase(unsigned int id) throw() { bool TestManager::clearPool() throw() { if (!tests()) return false; for (test_pool_it it = a_testPool.begin(); it != a_testPool.end(); it++) delete it->second; + // TODO: stop the possible command threads or there will be a core dump + a_testPool.clear(); a_sessionIdTestCaseMap.clear(); a_currentTestIt = a_testPool.end(); @@ -251,7 +300,7 @@ bool TestManager::nextTestCase() throw() { while (true) { // Limit for in-progress test cases: - if (a_inProgressCount >= a_inProgressLimit) { + if (getInProgressCount() >= a_inProgressLimit) { LOGDEBUG(anna::Logger::debug(anna::functions::asString("TestManager next case ignored (over in-progress count limit: %llu)", a_inProgressLimit), ANNA_FILE_LOCATION)); return true; // wait next tick to release OTA test cases } @@ -271,13 +320,13 @@ bool TestManager::nextTestCase() throw() { } else { LOGWARNING( - std::string nolimit = (a_poolRepeats != -1) ? "":" [no limit]"; - anna::Logger::warning(anna::functions::asString("Testing pool cycle %d completed (repeats configured: %d%s). Restarting for the %s cycle", a_poolCycle, a_poolRepeats, nolimit.c_str(), (a_poolRepeats == a_poolCycle) ? "last":"next"), ANNA_FILE_LOCATION); + std::string nolimit = (a_poolRepeats != -1) ? "":" [no limit]"; + anna::Logger::warning(anna::functions::asString("Testing pool cycle %d completed (repeats configured: %d%s). Restarting for the %s cycle", a_poolCycle, a_poolRepeats, nolimit.c_str(), (a_poolRepeats == a_poolCycle) ? "last":"next"), ANNA_FILE_LOCATION); ); a_poolCycle++; //a_currentTestIt = a_testPool.begin(); return true; // avoids infinite loop: if the cycle takes less time than test cases completion, below reset never will turns state - // into Initialized and this while will be infinite. It is preferable to wait one tick when the cycle is completed. + // into Initialized and this while will be infinite. It is preferable to wait one tick when the cycle is completed. } } @@ -289,8 +338,8 @@ bool TestManager::nextTestCase() throw() { if (a_currentTestIt->second->getState() != TestCase::State::InProgress) { a_currentTestIt->second->process(); return true; // is not probably to reach still In-Progress test cases from previous cycles due to the whole - // time for complete the test cases pool regarding the single test case lifetime. You shouldn't - // forget to programm a test case timeout with a reasonable value + // time for complete the test cases pool regarding the single test case lifetime. You shouldn't + // forget to programm a test case timeout with a reasonable value } } } @@ -401,7 +450,7 @@ throw() { if (a_poolRepeats) result->createAttribute("PoolRepeats", a_poolRepeats); else result->createAttribute("PoolRepeats", "disabled"); result->createAttribute("PoolCycle", a_poolCycle); - result->createAttribute("InProgressCount", a_inProgressCount); + a_statSummary.asXML(result); if (a_inProgressLimit == UINT_MAX) result->createAttribute("InProgressLimit", ""); else diff --git a/example/diameter/launcher/testing/TestManager.hpp b/example/diameter/launcher/testing/TestManager.hpp index 6a292e4..2f61501 100644 --- a/example/diameter/launcher/testing/TestManager.hpp +++ b/example/diameter/launcher/testing/TestManager.hpp @@ -19,6 +19,7 @@ // Process #include +#include namespace anna { @@ -51,6 +52,24 @@ typedef std::map::iterator test_pool Timer Manager for testing system */ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleton { + + // Statistics summary: + class StatSummary { + + unsigned int a_initializedTcs; + unsigned int a_inprogressTcs; + unsigned int a_failedTcs; + unsigned int a_sucessTcs; + + public: + void newTCState(const TestCase::State::_v beginState, const TestCase::State::_v endState) throw(); + void clear() throw(); + unsigned int getInProgressCount() const throw() { return a_inprogressTcs; } + anna::xml::Node* asXML(anna::xml::Node* parent) const throw(); + }; + + + typedef anna::Recycler timer_container; anna::timex::Engine* a_timeController; @@ -65,7 +84,6 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto test_pool_it a_currentTestIt; int a_poolRepeats; // repeat pool N times int a_poolCycle; // current cycle, from 1 to N - unsigned int a_inProgressCount; unsigned int a_inProgressLimit; // limit load to have this value // Test clock @@ -81,6 +99,8 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto std::map a_sessionIdTestCaseMap; // stores used Session-Id values within a test case. // No other can use them, but a test case could use more than one. + StatSummary a_statSummary; // general statistics + TestManager(); TestManager(const TestManager&); @@ -89,6 +109,7 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto void cancelTimer(TestTimer*) throw(); void release(anna::timex::TimeEvent*) throw(); + public: void registerSessionId(const std::string &sessionId, const TestCase *testCase) throw(anna::RuntimeException); @@ -115,8 +136,7 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto int getPoolRepeats() const throw() { return a_poolRepeats; } int getPoolCycle() const throw() { return a_poolCycle; } - unsigned int getInProgressCount() const throw() { return a_inProgressCount; } - void setInProgressCountDelta(unsigned int delta) throw() { a_inProgressCount += delta; } + unsigned int getInProgressCount() const throw() { return a_statSummary.getInProgressCount(); } unsigned int getInProgressLimit() const throw() { return a_inProgressLimit; } void setInProgressLimit(unsigned int limit) throw() { a_inProgressLimit = limit; } // 0 = UINT_MAX (no limit) @@ -132,6 +152,11 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto anna::xml::Node* asXML(anna::xml::Node* parent) const throw(); std::string asXMLString() const throw(); + // stats + void tcsStateStats(const TestCase::State::_v beginState, const TestCase::State::_v endState) throw() { + a_statSummary.newTCState(beginState, endState); + } + friend class anna::Singleton ; friend class TestStepTimeout; // createTimer