From: Eduardo Ramos Testillano Date: Mon, 23 Nov 2015 14:47:19 +0000 (+0100) Subject: Simplify parsed variables. Improve dump configuration for test report by state type X-Git-Tag: REFACTORING_TESTING_LIBRARY~79 X-Git-Url: https://git.teslayout.com/public/public/public/?a=commitdiff_plain;h=0a29aaeb86cc31d808f88d8c9a046f5858ed3089;p=anna.git Simplify parsed variables. Improve dump configuration for test report by state type --- diff --git a/example/diameter/launcher/Launcher.cpp b/example/diameter/launcher/Launcher.cpp index a6e5ab9..4219399 100644 --- a/example/diameter/launcher/Launcher.cpp +++ b/example/diameter/launcher/Launcher.cpp @@ -1340,19 +1340,6 @@ std::string Launcher::help() const throw() { result += "\n Test cases reports are not dumped on process context (too many information in general)."; result += "\n The report contains context information in every moment: this operation acts as a snapshot."; result += "\n"; - result += "\n test|report|[|[yes]|no]"; - result += "\n"; - result += "\n Every time a test case is finished, its xml representation will be dump on a file under"; - result += "\n the execution directory (or the one configured in process command-line 'tmDir') with"; - result += "\n the name 'cycle-.testcase-.xml'. The third parameter represents"; - result += "\n the test result condition to dump the information. By default, neither failed nor successful"; - result += "\n test cases are written to avoid IO overload (in most of cases not all the tests are going to"; - result += "\n fail and you could enable only such dumps, but you can enable both types if you want)."; - result += "\n Better, you can use the 'look' operation to see the desired report(s) and its result state,"; - result += "\n in case that you miss the xml files or you disabled all the dumps during testing."; - result += "\n"; - result += "\n test|report-hex[|[yes]|no] Reports could include the diameter messages in hexadecimal format. Disabled by default."; - result += "\n"; result += "\n test|interact|amount|id Makes interactive a specific test case id. The amount is the margin of execution steps"; result += "\n to be done. Normally, we will execute 'test|interact|0|', which means that"; result += "\n the test case is selected to be interactive, but no step is executed. Then you have to"; @@ -1377,8 +1364,24 @@ std::string Launcher::help() const throw() { result += "\n"; result += "\n test|summary Test manager general report (number of test cases, counts by state, global configuration,"; result += "\n forced in-progress limitation, reports visibility, etc.). Be careful when you have reports"; - result += "\n enabled because the programmed test cases dumps could be heavy (try to disable both failed"; - result += "\n and successful dumps)."; + result += "\n enabled because the programmed test cases dumps could be heavy (anyway you could enable the"; + result += "\n dumps separately, for any of the possible states: Initialized, InProgress, Failed, Success)."; + result += "\n"; + result += "\n test|report|[|[yes]|no]"; + result += "\n"; + result += "\n Enables/disables report generation for a certain test case state: initialized, in-progress,"; + result += "\n failed or success. This applies to report summary (former described operation) and automatic"; + result += "\n dumps during testing where only failed or success states will appear: every time a test case"; + result += "\n is finished its xml representation will be dump on a file under the execution directory (or"; + result += "\n the one configured in process command-line 'tmDir') with the name:"; + result += "\n"; + result += "\n 'cycle-.testcase-.xml'."; + result += "\n"; + result += "\n By default, all the states are disabled to avoid IO overload. In most of cases not all the"; + result += "\n tests are going to fail and you could enable only such failed dumps, but you can enable all"; + result += "\n the types if you want (use reserved word 'all' for this)."; + result += "\n"; + result += "\n test|report-hex[|[yes]|no] Reports could include the diameter messages in hexadecimal format. Disabled by default."; result += "\n"; result += "\n"; result += "\nUSING OPERATIONS INTERFACE"; @@ -1779,7 +1782,8 @@ void Launcher::eventOperation(const std::string &operation, std::string &respons // test|next[|] Forces the execution of the next test case(s) without waiting for test manager tick ... // test|ip-limit[|amount] In-progress limit of test cases. No new test cases will be launched over this value ... // test|repeats| Restarts the whole programmed test list when finished the amount number of times ... - // test|report|[|[yes]|no] Every time a test case is finished a report file in xml format will be created under ... + // test|report|[|[yes]|no] + // Enables/disables report generation for a certain test case state: initialized, in-progress ... // test|report-hex[|[yes]|no] Reports could include the diameter messages in hexadecimal format. Disabled by default. // test|goto| Updates current test pointer position. // test|look[|id] Show programmed test case for id provided, current when missing ... @@ -1852,16 +1856,22 @@ void Launcher::eventOperation(const std::string &operation, std::string &respons if (numParams > 3) throw anna::RuntimeException("Wrong body content format on HTTP Request. Use 'help' management command to see more information.", ANNA_FILE_LOCATION); - if(param2 != "failed" && param2 != "success") + if(param2 != "initialized" && param2 != "in-progress" && param2 != "failed" && param2 != "success" && param2 != "all") throw anna::RuntimeException("Wrong body content format on HTTP Request. Use 'help' management command to see more information.", ANNA_FILE_LOCATION); if(param3 == "") param3 = "yes"; bool enable = (param3 == "yes"); - if(param2 == "success") - testManager.setDumpSuccessfulReports(enable); - else + if(param2 == "initialized") + testManager.setDumpInitializedReports(enable); + else if(param2 == "in-progress") + testManager.setDumpInProgressReports(enable); + else if(param2 == "failed") testManager.setDumpFailedReports(enable); + else if(param2 == "success") + testManager.setDumpSuccessReports(enable); + else // all + testManager.setDumpAllReports(enable); opt_response_content += (enable ? "report enabled " : "report disabled "); opt_response_content += "for tests in '"; diff --git a/example/diameter/launcher/resources/scripts/clone.sh b/example/diameter/launcher/resources/scripts/clone.sh index 318e681..784185a 100755 --- a/example/diameter/launcher/resources/scripts/clone.sh +++ b/example/diameter/launcher/resources/scripts/clone.sh @@ -12,9 +12,12 @@ MAX_NUMBER_GROUPS=400 ############# _exit (){ echo - echo $1 + echo -e $1 echo + # Background jobs: + kill -9 $(jobs -p) + # Cleanup: rm -rf $CLONE_WKDIR @@ -33,16 +36,14 @@ usage() { echo " ADML directory: path to the ADML directory where 'operation.sh' script is used to load the test cases." echo " testcase variable file: path to the testcase file with parseable variables, for example '/testcase.txt.msk'." echo " xml files within this test case, must exist with aditional .msk extension in the same directory." + echo echo " Currently, 9 variables are supported to be replaced:" - echo " __TESTID__ : sequence number. For example if you provide 1 to 20 for this script, it will be 1 .. 20" - echo " __SEQ8__ : 8-digit number corresponding to the sequence number (00000001 for 1, and so on)" - echo " __MSISDN9__: 9-digit msisdn" - echo " __IPV4HEX__: hexadecimal IPv4 address, for example: 00000001" - echo " __IPV4__ : IPv4 address, for example: 0.0.0.1" - echo " __SGX1_SUFFIX__: 1<8-digit sequence>" - echo " __SRX1_SUFFIX__: \"" - echo " __SGX2_SUFFIX__: 2<8-digit sequence>" - echo " __SRX2_SUFFIX__: \"" + echo + echo " * __TESTID__ : sequence number. For example if you provide 1 to 20 for this script, it will be 1 .. 20" + echo " * __SEQ8__ : 8-digit number corresponding to the sequence number (00000001 for 1, and so on)" + echo " * __IPV4HEX__: hexadecimal IPv4 address, for example: 00000001" + echo " * __IPV4__ : IPv4 address, for example: 0.0.0.1" + echo echo " Edit the variables section to add more special values if you need them." echo echo " initial sequence: initial sequence number to parse over variables." @@ -91,28 +92,17 @@ clone() { ################################ EDIT THIS SECTION IF YOU NEED NEW SPECIAL VARIABLES ################################ testid=$sequence seq8=$(printf "%08d" $testid) - msisdn9=6${seq8} ipv4hex=$seq8 #ipv4=$(for i in $(echo $seq8 | sed 's/\(..\)/0x\1 /g'); do printf "%d." $i; done | sed 's/\.$/\n/') # Numbers beginning with "0" are treated as octal (i.e. base-8): we would have 'invalid octal number with 08 and 09' # Solution: convert to base-10 in this way: $((10#$i)) ipv4=$(for i in $(echo $seq8 | sed 's/\(..\)/ \1 /g'); do printf "%d." $((10#$i)); done | sed 's/\.$/\n/') - # Sessions - sgx1_suffix=1$seq8 - srx1_suffix=1$seq8 - sgx2_suffix=2$seq8 - srx2_suffix=2$seq8 local target=$wkdir/values.${1} echo "__TESTID__=$testid" > $target echo "__SEQ8__=$seq8" >> $target - echo "__MSISDN9__=$msisdn9" >> $target echo "__IPV4HEX__=$ipv4hex" >> $target echo "__IPV4__=$ipv4" >> $target - echo "__SGX1_SUFFIX__=$sgx1_suffix" >> $target - echo "__SRX1_SUFFIX__=$srx1_suffix" >> $target - echo "__SGX2_SUFFIX__=$sgx2_suffix" >> $target - echo "__SRX2_SUFFIX__=$srx2_suffix" >> $target source $target #rm $target ######################################### END SPECIAL VARIABLES SECTION ######################################### @@ -227,7 +217,9 @@ do #bn_dn_r_file=`basename $dn_r_file` #echo "Programming group $bn_dn_r_file ..." $OPERATION -f $r_file >/dev/null + res=$? #$OPERATION -f $r_file > $dn_r_file/result.txt + [ $res -ne 0 ] && _exit "Exception detected programming a testcase:\n\n`cat $r_file`" rm -rf $dn_r_file & done diff --git a/example/diameter/launcher/resources/scripts/operation_signal.sh b/example/diameter/launcher/resources/scripts/operation_signal.sh index 1be6fa9..c6c15be 100755 --- a/example/diameter/launcher/resources/scripts/operation_signal.sh +++ b/example/diameter/launcher/resources/scripts/operation_signal.sh @@ -35,5 +35,8 @@ kill -s SIGUSR2 $PID # Detect EOF and print all except that last line: while [ -z "$(tail -1 sigusr2.out | grep ^EOF)" ]; do sleep 0.1; done -head --lines=-1 `pwd`/sigusr2.out +head --lines=-1 sigusr2.out +exception=$(grep exception sigusr2.out) +[ $? -eq 0 ] && _exit "(detected 'exception' within operation output)" +exit 0 diff --git a/example/diameter/launcher/testing/TestCase.cpp b/example/diameter/launcher/testing/TestCase.cpp index d69a61a..bf08b4e 100644 --- a/example/diameter/launcher/testing/TestCase.cpp +++ b/example/diameter/launcher/testing/TestCase.cpp @@ -133,7 +133,7 @@ void TestCase::setState(const State::_v &state) throw() { if (isFinished()) { if ((getState() == State::Failed) && (!testManager.getDumpFailedReports())) return; - if ((getState() == State::Success) && (!testManager.getDumpSuccessfulReports())) return; + if ((getState() == State::Success) && (!testManager.getDumpSuccessReports())) return; // report file name: cycle-.testcase-.xml // FORMAT: We tabulate the cycle and test case in order to ease ordering of files by mean ls: diff --git a/example/diameter/launcher/testing/TestManager.cpp b/example/diameter/launcher/testing/TestManager.cpp index e9cca9b..01487fe 100644 --- a/example/diameter/launcher/testing/TestManager.cpp +++ b/example/diameter/launcher/testing/TestManager.cpp @@ -82,8 +82,12 @@ TestManager::TestManager() : anna::timex::TimeEventObserver("TestManager") { a_timeController = NULL; a_reportsDirectory = "./"; - a_dumpSuccessfulReports = false; + + a_dumpInProgressReports = false; + a_dumpInitializedReports = false; a_dumpFailedReports = false; + a_dumpSuccessReports = false; + a_dumpHexMessages = false; a_synchronousAmount = 1; a_poolRepeats = 0; // repeat disabled by default @@ -461,8 +465,10 @@ throw() { result->createAttribute("InProgressLimit", ""); else result->createAttribute("InProgressLimit", a_inProgressLimit); - result->createAttribute("DumpSuccessfulReports", (a_dumpSuccessfulReports ? "yes":"no")); + result->createAttribute("DumpInitializedReports", (a_dumpInitializedReports ? "yes":"no")); + result->createAttribute("DumpInProgressReports", (a_dumpInProgressReports ? "yes":"no")); result->createAttribute("DumpFailedReports", (a_dumpFailedReports ? "yes":"no")); + result->createAttribute("DumpSuccessReports", (a_dumpSuccessReports ? "yes":"no")); result->createAttribute("DumpHexMessages", (a_dumpHexMessages ? "yes":"no")); result->createAttribute("ReportsDirectory", a_reportsDirectory); if (a_clock) { @@ -476,8 +482,10 @@ throw() { if (poolSize != 0) { anna::xml::Node* testCases = result->createChild("TestCases"); for (test_pool_it it = a_testPool.begin(); it != a_testPool.end(); it++) { - if (((*it).second->getState() == TestCase::State::Success) && (!getDumpSuccessfulReports())) continue; + if (((*it).second->getState() == TestCase::State::Initialized) && (!getDumpInitializedReports())) continue; + if (((*it).second->getState() == TestCase::State::InProgress) && (!getDumpInProgressReports())) continue; if (((*it).second->getState() == TestCase::State::Failed) && (!getDumpFailedReports())) continue; + if (((*it).second->getState() == TestCase::State::Success) && (!getDumpSuccessReports())) continue; (*it).second->asXML(testCases); } } diff --git a/example/diameter/launcher/testing/TestManager.hpp b/example/diameter/launcher/testing/TestManager.hpp index 43c5340..af40fbf 100644 --- a/example/diameter/launcher/testing/TestManager.hpp +++ b/example/diameter/launcher/testing/TestManager.hpp @@ -76,7 +76,7 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto // reports std::string a_reportsDirectory; - bool a_dumpSuccessfulReports, a_dumpFailedReports; + bool a_dumpInitializedReports, a_dumpInProgressReports, a_dumpFailedReports, a_dumpSuccessReports; bool a_dumpHexMessages; // Pool of test cases @@ -123,11 +123,23 @@ class TestManager : public anna::timex::TimeEventObserver, public anna::Singleto void setDumpHex(bool dh) throw() { a_dumpHexMessages = dh; } bool getDumpHex() const throw() { return a_dumpHexMessages; } - void setDumpSuccessfulReports(bool dsr) throw() { a_dumpSuccessfulReports = dsr; } - void setDumpFailedReports(bool dfr) throw() { a_dumpFailedReports = dfr; } - bool getDumpSuccessfulReports() const throw() { return a_dumpSuccessfulReports; } + + void setDumpInitializedReports(bool enable) throw() { a_dumpInitializedReports = enable; } + void setDumpInProgressReports(bool enable) throw() { a_dumpInProgressReports = enable; } + void setDumpFailedReports(bool enable) throw() { a_dumpFailedReports = enable; } + void setDumpSuccessReports(bool enable) throw() { a_dumpSuccessReports = enable; } + void setDumpAllReports(bool enable) throw() { + setDumpInitializedReports(enable); + setDumpInProgressReports(enable); + setDumpFailedReports(enable); + setDumpSuccessReports(enable); + } + + bool getDumpInitializedReports() const throw() { return a_dumpInitializedReports; } + bool getDumpInProgressReports() const throw() { return a_dumpInProgressReports; } bool getDumpFailedReports() const throw() { return a_dumpFailedReports; } + bool getDumpSuccessReports() const throw() { return a_dumpSuccessReports; } // Helper to calculate time interval and synchronous amount of execution tests to guarantee the input rate (tests per second) // through the time manager which has a minimum resolution of ADML minimum resolution. The first call to this method will