From 983eaadca6cfae987be3453853d75bb9bce04487 Mon Sep 17 00:00:00 2001 From: Eduardo Ramos Testillano Date: Mon, 16 Nov 2015 21:59:49 +0100 Subject: [PATCH] Optimize clone procedure --- example/diameter/launcher/Launcher.cpp | 6 +- .../launcher/resources/scripts/clone.sh | 212 ++++++++++++++++++ .../resources/scripts/operation_signal.sh | 6 +- .../diameter/launcher/testing/TestCase.cpp | 8 +- .../diameter/launcher/testing/TestManager.cpp | 1 + 5 files changed, 224 insertions(+), 9 deletions(-) create mode 100755 example/diameter/launcher/resources/scripts/clone.sh diff --git a/example/diameter/launcher/Launcher.cpp b/example/diameter/launcher/Launcher.cpp index 1b7a7ee..45f51a9 100644 --- a/example/diameter/launcher/Launcher.cpp +++ b/example/diameter/launcher/Launcher.cpp @@ -802,7 +802,7 @@ void Launcher::resetCounters() throw() { void Launcher::signalUSR2() throw(anna::RuntimeException) { LOGNOTICE( - std::string msg = "Captured signal SIGUSR2. Reading tasks at '"; + std::string msg = "Captured signal SIGUSR2. Reading tasks at '"; msg += SIGUSR2_TASKS_INPUT_FILENAME; msg += "' (results will be written at '"; msg += SIGUSR2_TASKS_OUTPUT_FILENAME; @@ -817,7 +817,6 @@ void Launcher::signalUSR2() throw(anna::RuntimeException) { std::ofstream out_file(SIGUSR2_TASKS_OUTPUT_FILENAME); if(!in_file.is_open()) throw RuntimeException("Unable to read tasks", ANNA_FILE_LOCATION); - if(!out_file.is_open()) throw RuntimeException("Unable to write tasks", ANNA_FILE_LOCATION); while(getline(in_file, line)) { @@ -837,6 +836,7 @@ void Launcher::signalUSR2() throw(anna::RuntimeException) { } in_file.close(); + out_file << "EOF\n"; out_file.close(); } @@ -1403,6 +1403,8 @@ std::string Launcher::help() const throw() { result += "\n and then see the results:"; result += "\n cat "; result += SIGUSR2_TASKS_OUTPUT_FILENAME; result += "\n"; + result += "\n (this file is ended with EOF final line, useful managing huge batch files to ensure the job completion)"; + result += "\n"; result += "\nYou could place more than one line (task) in the input file. Output reports will be appended in that"; result += "\n case over the output file. Take into account that all the content of the task file will be executed"; result += "\n sinchronously by the process. If you are planning traffic load, better use the asynchronous http"; diff --git a/example/diameter/launcher/resources/scripts/clone.sh b/example/diameter/launcher/resources/scripts/clone.sh new file mode 100755 index 0000000..c6560f3 --- /dev/null +++ b/example/diameter/launcher/resources/scripts/clone.sh @@ -0,0 +1,212 @@ +#!/bin/bash + +############# +# VARIABLES # +############# +CLONE_WKDIR=$(mktemp -d) +MAX_NUMBER_GROUPS=400 +# (depends on the machine) + +############# +# FUNCTIONS # +############# +_exit (){ + echo + echo $1 + echo + + # Cleanup: + rm -rf $CLONE_WKDIR + + rc=1 + [ -n "$2" ] && rc=$2 + exit $rc +} + +sig_handler() { + _exit "Script interrupted. Cleanup & exit ..." +} + +usage() { + echo "Usage: $0 [number of groups]" + echo + echo " ADML directory: path to the ADML directory where 'operation.sh' script is used to load the test cases." + echo " testcase variable file: path to the testcase file with parseable variables, for example '/testcase.txt.msk'." + echo " xml files within this test case, must exist with aditional .msk extension in the same directory." + echo " Currently, 9 variables are supported to be replaced:" + echo " __TESTID__ : sequence number. For example if you provide 1 to 20 for this script, it will be 1 .. 20" + echo " __SEQ8__ : 8-digit number corresponding to the sequence number (00000001 for 1, and so on)" + echo " __MSISDN9__: 9-digit msisdn" + echo " __IPV4HEX__: hexadecimal IPv4 address, for example: 00000001" + echo " __IPV4__ : IPv4 address, for example: 0.0.0.1" + echo " __SGX1_SUFFIX__: 1<8-digit sequence>" + echo " __SRX1_SUFFIX__: \"" + echo " __SGX2_SUFFIX__: 2<8-digit sequence>" + echo " __SRX2_SUFFIX__: \"" + echo + echo " initial sequence: initial sequence number to parse over variables." + echo " final sequence: final sequence number to parse over variables." + echo " number of groups: number of background group jobs to clone the provided sequence range." + echo " By default, it will be the number of items divided by 25, with a maximum of $MAX_NUMBER_GROUPS." + echo + echo + _exit +} + +children () { + bash_pid=$$ + children=`ps -eo ppid | grep -w $bash_pid` + echo $children | wc -w +} + +# $1: sequence number; $2: file to parse(will be updated); $3: result file +parse_file() { + local vars=$(egrep -o '__((_*[A-Z]*[0-9]*)*)__' $2 | cut -d: -f2 | sort -u) + local file=$2 + cp $2 $3 + file=$3 + for pvar in $vars + do + eval pvalue=\$$pvar + sed -i 's/'$pvar'/'$pvalue'/g' $file + done +} + +# $1: sequence number; $2: working directory +clone() { + local sequence=$1 + local wkdir=$2 + mkdir -p $wkdir + + # Values: + testid=$sequence + seq8=$(printf "%08d" $testid) + msisdn9=6${seq8} + ipv4hex=$seq8 + #ipv4=$(for i in $(echo $seq8 | sed 's/\(..\)/0x\1 /g'); do printf "%d." $i; done | sed 's/\.$/\n/') + # Numbers beginning with "0" are treated as octal (i.e. base-8): we would have 'invalid octal number with 08 and 09' + # Solution: convert to base-10 in this way: $((10#$i)) + ipv4=$(for i in $(echo $seq8 | sed 's/\(..\)/ \1 /g'); do printf "%d." $((10#$i)); done | sed 's/\.$/\n/') + # Sessions + sgx1_suffix=1$seq8 + srx1_suffix=1$seq8 + sgx2_suffix=2$seq8 + srx2_suffix=2$seq8 + + local target=$wkdir/values.${1} + echo "__TESTID__=$testid" > $target + echo "__SEQ8__=$seq8" >> $target + echo "__MSISDN9__=$msisdn9" >> $target + echo "__IPV4HEX__=$ipv4hex" >> $target + echo "__IPV4__=$ipv4" >> $target + echo "__SGX1_SUFFIX__=$sgx1_suffix" >> $target + echo "__SRX1_SUFFIX__=$srx1_suffix" >> $target + echo "__SGX2_SUFFIX__=$sgx2_suffix" >> $target + echo "__SRX2_SUFFIX__=$srx2_suffix" >> $target + source $target + #rm $target + + # Parse template files: + parse_file $sequence $TESTCASE_TEMPLATE $wkdir/testcase.txt.$sequence + for file in $(grep sendxml $TESTCASE_TEMPLATE | cut -d\| -f4 | sed 's/\.xml$/.xml.msk/') + do + xml=`basename $file .msk` + parse_file $sequence $TESTCASE_TEMPLATE_DIR/$file $wkdir/${xml}.${sequence} + new_file=`readlink -f $wkdir/${xml}.${sequence}` + sed -i 's|'$xml'|'$new_file'|g' $wkdir/testcase.txt.$sequence + done + cat $wkdir/testcase.txt.$sequence >> $wkdir/testcase.txt + rm $wkdir/testcase.txt.$sequence +} + +# $1: group number; $2: initial subrange value; $3: final subrange value +clone_group() { + for i in `seq $2 $3` + do + clone $i $CLONE_WKDIR/$1 + #sleep 0.01 + done +} + +############# +# EXECUTION # +############# +trap sig_handler SIGINT +trap sig_handler SIGTERM + +ADML_DIR=$1 +TESTCASE_TEMPLATE=`readlink -f $2` +CLONE_SEQ_BEGIN=$3 +CLONE_SEQ_END=$4 +N_GROUPS=$5 + +[ "$4" = "" ] && usage + +# Auxiliary: +OPERATION=$ADML_DIR/operation.sh +[ ! -f $OPERATION ] && _exit "Missing '$OPERATION' file !!" +[ ! -f $TESTCASE_TEMPLATE ] && _exit "Missing '$TESTCASE_TEMPLATE' testcase template file !!" + +TESTCASE_TEMPLATE_DIR=`dirname $TESTCASE_TEMPLATE` +N_ITEMS=$((CLONE_SEQ_END - CLONE_SEQ_BEGIN + 1)) +if [ -z "$N_GROUPS" ] +then + N_GROUPS=$((N_ITEMS/25)) + [ $N_GROUPS -gt $MAX_NUMBER_GROUPS ] && N_GROUPS=$MAX_NUMBER_GROUPS + [ $N_GROUPS -eq 0 ] && N_GROUPS=1 +fi +GROUPS_SIZE=$((N_ITEMS/N_GROUPS)) +if [ "$GROUPS_SIZE" -eq 0 ] +then + echo "Assuming minimum allowed number of groups: $N_ITEMS" + GROUPS_SIZE=1 + N_GROUPS=$N_ITEMS +fi + +# Start cloning: +timestamp_begin=$(echo "scale=3 ; $(date '+%s') + $(date '+%N') / 1000000000" | bc) +children_before=$(children) +offset=0 +echo "Please be patient, this may take a while ..." +$OPERATION "test|clear" +for group in `seq 1 $((N_GROUPS+1))` +do + n_begin=$((CLONE_SEQ_BEGIN + offset)) + n_end=$((n_begin + GROUPS_SIZE - 1)) + [ $n_end -gt $CLONE_SEQ_END ] && n_end=$CLONE_SEQ_END + [ $n_end -lt $n_begin ] && break + clone_group $group $n_begin $n_end & + offset=$((group * GROUPS_SIZE)) +done +children_now=$(children) + +# Wait background jobs to finish: +while true +do + sleep 1 + [ $(children) -eq $children_before ] && break +done + +# Programming: +echo -n "Programming .." +for file in $(ls $CLONE_WKDIR/*/testcase.txt) +do + echo -n . + r_file=`readlink -f $file` + dn_r_file=`dirname $r_file` + #bn_dn_r_file=`basename $dn_r_file` + #echo "Programming group $bn_dn_r_file ..." + $OPERATION -f $r_file >/dev/null + #$OPERATION -f $r_file > $dn_r_file/result.txt + rm -rf $dn_r_file & +done + +timestamp_end=$(echo "scale=3 ; $(date '+%s') + $(date '+%N') / 1000000000" | bc) +echo +lasted=$(echo "scale=3 ; $timestamp_end - $timestamp_begin" | bc) +echo "Total Lasted $lasted seconds" +echo "Programming speed: $(echo "$N_ITEMS/$lasted" | bc) tests per second" + +# Finish: +_exit "Done!" 0 + diff --git a/example/diameter/launcher/resources/scripts/operation_signal.sh b/example/diameter/launcher/resources/scripts/operation_signal.sh index 4db01a5..85b3111 100755 --- a/example/diameter/launcher/resources/scripts/operation_signal.sh +++ b/example/diameter/launcher/resources/scripts/operation_signal.sh @@ -30,7 +30,7 @@ else fi 0> sigusr2.out kill -s SIGUSR2 $PID -count=0 -while [ ! -s sigusr2.out -a $count -lt 500 ]; do sleep 0.01; count=$((count+1)); done -cat `pwd`/sigusr2.out +# Detect EOF and print all except that last line: +while [ -z "$(tail -1 sigusr2.out | grep ^EOF)" ]; do sleep 0.1; done +head --lines=-1 `pwd`/sigusr2.out diff --git a/example/diameter/launcher/testing/TestCase.cpp b/example/diameter/launcher/testing/TestCase.cpp index 3d91f32..2817248 100644 --- a/example/diameter/launcher/testing/TestCase.cpp +++ b/example/diameter/launcher/testing/TestCase.cpp @@ -341,9 +341,9 @@ void TestCase::addWait(bool fromEntity, if (!step) step = new TestStepWait(this); step->setCondition(fromEntity, code, bitR, usedHopByHop, applicationId, sessionId, resultCode, msisdn, imsi, serviceContextId); - LOGWARNING( + LOGINFORMATION( if (hasSameCondition(step->getCondition())) - anna::Logger::warning(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION); + anna::Logger::information(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION); ); addStep(step); @@ -355,9 +355,9 @@ void TestCase::addWaitRegexp(bool fromEntity, const std::string ®exp) throw(a TestStepWait *step = new TestStepWait(this); step->setCondition(fromEntity, regexp); - LOGWARNING( + LOGINFORMATION( if (hasSameCondition(step->getCondition())) - anna::Logger::warning(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION); + anna::Logger::information(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION); ); addStep(step); diff --git a/example/diameter/launcher/testing/TestManager.cpp b/example/diameter/launcher/testing/TestManager.cpp index 2c03b4d..bd9bbfc 100644 --- a/example/diameter/launcher/testing/TestManager.cpp +++ b/example/diameter/launcher/testing/TestManager.cpp @@ -262,6 +262,7 @@ bool TestManager::clearPool() throw() { a_currentTestIt = a_testPool.end(); a_poolCycle = 1; configureTTPS(0); // stop + a_statSummary.clear(); return true; } -- 2.20.1