From: Eduardo Ramos Testillano <eduardo.ramos.testillano@ericsson.com>
Date: Mon, 16 Nov 2015 20:59:49 +0000 (+0100)
Subject: Optimize clone procedure
X-Git-Tag: REFACTORING_TESTING_LIBRARY~86
X-Git-Url: https://git.teslayout.com/public/public/public/?a=commitdiff_plain;h=983eaadca6cfae987be3453853d75bb9bce04487;p=anna.git

Optimize clone procedure
---

diff --git a/example/diameter/launcher/Launcher.cpp b/example/diameter/launcher/Launcher.cpp
index 1b7a7ee..45f51a9 100644
--- a/example/diameter/launcher/Launcher.cpp
+++ b/example/diameter/launcher/Launcher.cpp
@@ -802,7 +802,7 @@ void Launcher::resetCounters() throw() {
 
 void Launcher::signalUSR2() throw(anna::RuntimeException) {
   LOGNOTICE(
-      std::string msg = "Captured signal SIGUSR2. Reading tasks at '";
+  std::string msg = "Captured signal SIGUSR2. Reading tasks at '";
   msg += SIGUSR2_TASKS_INPUT_FILENAME;
   msg += "' (results will be written at '";
   msg += SIGUSR2_TASKS_OUTPUT_FILENAME;
@@ -817,7 +817,6 @@ void Launcher::signalUSR2() throw(anna::RuntimeException) {
   std::ofstream out_file(SIGUSR2_TASKS_OUTPUT_FILENAME);
 
   if(!in_file.is_open()) throw RuntimeException("Unable to read tasks", ANNA_FILE_LOCATION);
-
   if(!out_file.is_open()) throw RuntimeException("Unable to write tasks", ANNA_FILE_LOCATION);
 
   while(getline(in_file, line)) {
@@ -837,6 +836,7 @@ void Launcher::signalUSR2() throw(anna::RuntimeException) {
   }
 
   in_file.close();
+  out_file << "EOF\n";
   out_file.close();
 }
 
@@ -1403,6 +1403,8 @@ std::string Launcher::help() const throw() {
   result += "\n    and then see the results:";
   result += "\n   cat "; result += SIGUSR2_TASKS_OUTPUT_FILENAME;
   result += "\n";
+  result += "\n   (this file is ended with EOF final line, useful managing huge batch files to ensure the job completion)";
+  result += "\n";
   result += "\nYou could place more than one line (task) in the input file. Output reports will be appended in that";
   result += "\n case over the output file. Take into account that all the content of the task file will be executed";
   result += "\n sinchronously by the process. If you are planning traffic load, better use the asynchronous http";
diff --git a/example/diameter/launcher/resources/scripts/clone.sh b/example/diameter/launcher/resources/scripts/clone.sh
new file mode 100755
index 0000000..c6560f3
--- /dev/null
+++ b/example/diameter/launcher/resources/scripts/clone.sh
@@ -0,0 +1,212 @@
+#!/bin/bash
+
+#############
+# VARIABLES #
+#############
+CLONE_WKDIR=$(mktemp -d)
+MAX_NUMBER_GROUPS=400
+# (depends on the machine)
+
+#############
+# FUNCTIONS #
+#############
+_exit (){
+  echo
+  echo $1
+  echo
+
+  # Cleanup:
+  rm -rf $CLONE_WKDIR
+
+  rc=1
+  [ -n "$2" ] && rc=$2
+  exit $rc
+}
+
+sig_handler() {
+  _exit "Script interrupted. Cleanup & exit ..."
+}
+
+usage() {
+  echo "Usage: $0 <ADML directory> <testcase variable file> <initial sequence> <final sequence> [number of groups]"
+  echo
+  echo "          ADML directory:         path to the ADML directory where 'operation.sh' script is used to load the test cases."
+  echo "          testcase variable file: path to the testcase file with parseable variables, for example '<directory>/testcase.txt.msk'."
+  echo "                                  xml files within this test case, must exist with aditional .msk extension in the same directory."
+  echo "                                  Currently, 9 variables are supported to be replaced:"
+  echo "                                    __TESTID__ : sequence number. For example if you provide 1 to 20 for this script, it will be 1 .. 20"
+  echo "                                    __SEQ8__   : 8-digit number corresponding to the sequence number (00000001 for 1, and so on)"
+  echo "                                    __MSISDN9__: 9-digit msisdn"
+  echo "                                    __IPV4HEX__: hexadecimal IPv4 address, for example: 00000001"
+  echo "                                    __IPV4__   : IPv4 address, for example: 0.0.0.1"
+  echo "                                    __SGX1_SUFFIX__: 1<8-digit sequence>"
+  echo "                                    __SRX1_SUFFIX__: \""
+  echo "                                    __SGX2_SUFFIX__: 2<8-digit sequence>"
+  echo "                                    __SRX2_SUFFIX__: \""
+  echo
+  echo "          initial sequence:       initial sequence number to parse over variables."
+  echo "          final sequence:         final sequence number to parse over variables."
+  echo "          number of groups:       number of background group jobs to clone the provided sequence range."
+  echo "                                  By default, it will be the number of items divided by 25, with a maximum of $MAX_NUMBER_GROUPS."
+  echo
+  echo
+  _exit
+}
+
+children () {
+  bash_pid=$$
+  children=`ps -eo ppid | grep -w $bash_pid`
+  echo $children | wc -w
+}
+
+# $1: sequence number; $2: file to parse(will be updated); $3: result file
+parse_file() {
+  local vars=$(egrep -o '__((_*[A-Z]*[0-9]*)*)__' $2 | cut -d: -f2 | sort -u)
+  local file=$2
+  cp $2 $3
+  file=$3
+  for pvar in $vars
+  do
+    eval pvalue=\$$pvar
+    sed -i 's/'$pvar'/'$pvalue'/g' $file
+  done
+}
+
+# $1: sequence number; $2: working directory
+clone() {
+  local sequence=$1
+  local wkdir=$2
+  mkdir -p $wkdir
+
+  # Values:
+  testid=$sequence
+  seq8=$(printf "%08d" $testid)
+  msisdn9=6${seq8}
+  ipv4hex=$seq8
+  #ipv4=$(for i in $(echo $seq8 | sed 's/\(..\)/0x\1 /g'); do printf "%d." $i; done | sed 's/\.$/\n/')
+  # Numbers beginning with "0" are treated as octal (i.e. base-8): we would have 'invalid octal number with 08 and 09'
+  # Solution: convert to base-10 in this way: $((10#$i))
+  ipv4=$(for i in $(echo $seq8 | sed 's/\(..\)/ \1 /g'); do printf "%d." $((10#$i)); done | sed 's/\.$/\n/')
+  # Sessions
+  sgx1_suffix=1$seq8
+  srx1_suffix=1$seq8
+  sgx2_suffix=2$seq8
+  srx2_suffix=2$seq8
+
+  local target=$wkdir/values.${1}
+  echo "__TESTID__=$testid" > $target
+  echo "__SEQ8__=$seq8" >> $target
+  echo "__MSISDN9__=$msisdn9" >> $target
+  echo "__IPV4HEX__=$ipv4hex" >> $target
+  echo "__IPV4__=$ipv4" >> $target
+  echo "__SGX1_SUFFIX__=$sgx1_suffix" >> $target
+  echo "__SRX1_SUFFIX__=$srx1_suffix" >> $target
+  echo "__SGX2_SUFFIX__=$sgx2_suffix" >> $target
+  echo "__SRX2_SUFFIX__=$srx2_suffix" >> $target
+  source $target
+  #rm $target
+
+  # Parse template files:
+  parse_file $sequence $TESTCASE_TEMPLATE $wkdir/testcase.txt.$sequence
+  for file in $(grep sendxml $TESTCASE_TEMPLATE | cut -d\| -f4 | sed 's/\.xml$/.xml.msk/')
+  do
+    xml=`basename $file .msk`
+    parse_file $sequence $TESTCASE_TEMPLATE_DIR/$file $wkdir/${xml}.${sequence}
+    new_file=`readlink -f $wkdir/${xml}.${sequence}`
+    sed -i 's|'$xml'|'$new_file'|g' $wkdir/testcase.txt.$sequence
+  done
+  cat $wkdir/testcase.txt.$sequence >> $wkdir/testcase.txt
+  rm $wkdir/testcase.txt.$sequence
+}
+
+# $1: group number; $2: initial subrange value; $3: final subrange value
+clone_group() {
+  for i in `seq $2 $3`
+  do
+    clone $i $CLONE_WKDIR/$1
+    #sleep 0.01
+  done
+}
+
+#############
+# EXECUTION #
+#############
+trap sig_handler SIGINT
+trap sig_handler SIGTERM
+
+ADML_DIR=$1
+TESTCASE_TEMPLATE=`readlink -f $2`
+CLONE_SEQ_BEGIN=$3
+CLONE_SEQ_END=$4
+N_GROUPS=$5
+
+[ "$4" = "" ] && usage
+
+# Auxiliary:
+OPERATION=$ADML_DIR/operation.sh
+[ ! -f $OPERATION ] && _exit "Missing '$OPERATION' file !!"
+[ ! -f $TESTCASE_TEMPLATE ] && _exit "Missing '$TESTCASE_TEMPLATE' testcase template file !!"
+
+TESTCASE_TEMPLATE_DIR=`dirname $TESTCASE_TEMPLATE`
+N_ITEMS=$((CLONE_SEQ_END - CLONE_SEQ_BEGIN + 1))
+if [ -z "$N_GROUPS" ]
+then
+  N_GROUPS=$((N_ITEMS/25))
+  [ $N_GROUPS -gt $MAX_NUMBER_GROUPS ] && N_GROUPS=$MAX_NUMBER_GROUPS
+  [ $N_GROUPS -eq 0 ] && N_GROUPS=1
+fi
+GROUPS_SIZE=$((N_ITEMS/N_GROUPS))
+if [ "$GROUPS_SIZE" -eq 0 ]
+then
+  echo "Assuming minimum allowed number of groups: $N_ITEMS"
+  GROUPS_SIZE=1
+  N_GROUPS=$N_ITEMS
+fi
+
+# Start cloning:
+timestamp_begin=$(echo "scale=3 ; $(date '+%s') + $(date '+%N') / 1000000000" | bc)
+children_before=$(children)
+offset=0
+echo "Please be patient, this may take a while ..."
+$OPERATION "test|clear"
+for group in `seq 1 $((N_GROUPS+1))`
+do
+  n_begin=$((CLONE_SEQ_BEGIN + offset))
+  n_end=$((n_begin + GROUPS_SIZE - 1))
+  [ $n_end -gt $CLONE_SEQ_END ] && n_end=$CLONE_SEQ_END
+  [ $n_end -lt $n_begin ] && break
+  clone_group $group $n_begin $n_end &
+  offset=$((group * GROUPS_SIZE))
+done
+children_now=$(children)
+
+# Wait background jobs to finish:
+while true
+do
+  sleep 1 
+  [ $(children) -eq $children_before ] && break
+done
+
+# Programming:
+echo -n "Programming .."
+for file in $(ls $CLONE_WKDIR/*/testcase.txt)
+do
+  echo -n .
+  r_file=`readlink -f $file`
+  dn_r_file=`dirname $r_file`
+  #bn_dn_r_file=`basename $dn_r_file`
+  #echo "Programming group $bn_dn_r_file ..."
+  $OPERATION -f $r_file >/dev/null
+  #$OPERATION -f $r_file > $dn_r_file/result.txt
+  rm -rf $dn_r_file &
+done
+
+timestamp_end=$(echo "scale=3 ; $(date '+%s') + $(date '+%N') / 1000000000" | bc)
+echo
+lasted=$(echo "scale=3 ; $timestamp_end - $timestamp_begin" | bc)
+echo "Total Lasted $lasted seconds"
+echo "Programming speed: $(echo "$N_ITEMS/$lasted" | bc) tests per second"
+
+# Finish:
+_exit "Done!" 0
+
diff --git a/example/diameter/launcher/resources/scripts/operation_signal.sh b/example/diameter/launcher/resources/scripts/operation_signal.sh
index 4db01a5..85b3111 100755
--- a/example/diameter/launcher/resources/scripts/operation_signal.sh
+++ b/example/diameter/launcher/resources/scripts/operation_signal.sh
@@ -30,7 +30,7 @@ else
 fi
 0> sigusr2.out
 kill -s SIGUSR2 $PID
-count=0
-while [ ! -s sigusr2.out -a $count -lt 500 ]; do sleep 0.01; count=$((count+1)); done
-cat `pwd`/sigusr2.out
+# Detect EOF and print all except that last line:
+while [ -z "$(tail -1 sigusr2.out | grep ^EOF)" ]; do sleep 0.1; done
+head --lines=-1 `pwd`/sigusr2.out
 
diff --git a/example/diameter/launcher/testing/TestCase.cpp b/example/diameter/launcher/testing/TestCase.cpp
index 3d91f32..2817248 100644
--- a/example/diameter/launcher/testing/TestCase.cpp
+++ b/example/diameter/launcher/testing/TestCase.cpp
@@ -341,9 +341,9 @@ void TestCase::addWait(bool fromEntity,
   if (!step) step = new TestStepWait(this);
   step->setCondition(fromEntity, code, bitR, usedHopByHop, applicationId, sessionId, resultCode, msisdn, imsi, serviceContextId);
 
-  LOGWARNING(
+  LOGINFORMATION(
     if (hasSameCondition(step->getCondition()))
-      anna::Logger::warning(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION);
+      anna::Logger::information(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION);
   );
 
   addStep(step);
@@ -355,9 +355,9 @@ void TestCase::addWaitRegexp(bool fromEntity, const std::string &regexp) throw(a
   TestStepWait *step = new TestStepWait(this);
   step->setCondition(fromEntity, regexp);
 
-  LOGWARNING(
+  LOGINFORMATION(
     if (hasSameCondition(step->getCondition()))
-      anna::Logger::warning(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION);
+      anna::Logger::information(anna::functions::asString("The same wait condition has already been programmed in this test case (%llu). Are you sure ?", a_id), ANNA_FILE_LOCATION);
   );
 
   addStep(step);
diff --git a/example/diameter/launcher/testing/TestManager.cpp b/example/diameter/launcher/testing/TestManager.cpp
index 2c03b4d..bd9bbfc 100644
--- a/example/diameter/launcher/testing/TestManager.cpp
+++ b/example/diameter/launcher/testing/TestManager.cpp
@@ -262,6 +262,7 @@ bool TestManager::clearPool() throw() {
   a_currentTestIt = a_testPool.end();
   a_poolCycle = 1;
   configureTTPS(0); // stop
+  a_statSummary.clear();
   return true;
 }