From 77105a89de724b0bc5b4b9c9f2844cb44db1de3e Mon Sep 17 00:00:00 2001 From: Eduardo Ramos Testillano Date: Fri, 26 Feb 2016 04:56:48 +0100 Subject: [PATCH] Support for Sy/eSy --- .../resources/scripts/tinyTestcase.sh | 202 ++++++++---------- example/diameter/pcapDecoder/tsharkDecoder.sh | 48 +++-- 2 files changed, 115 insertions(+), 135 deletions(-) diff --git a/example/diameter/launcher/resources/scripts/tinyTestcase.sh b/example/diameter/launcher/resources/scripts/tinyTestcase.sh index 08145fd..c4f4db9 100755 --- a/example/diameter/launcher/resources/scripts/tinyTestcase.sh +++ b/example/diameter/launcher/resources/scripts/tinyTestcase.sh @@ -19,83 +19,63 @@ _exit() { usage() { echo - echo "Usage: $0 \"[test_end_points]\" [timeout]" + echo "Usage: $0 [seconds_timeout]" echo echo " source_directory:" + echo " The source directory may contain .xml and .metadata files grouped by pairs and alphabetically classified," + echo " and also (and this is VERY IMPORTANT), a file called 'origin-hosts' containing a list of Origin-Host" + echo " values that are used from the point of view of the ADML testing system, together with the type of the" + echo " simulated ADML node (client/server):" echo - echo " The source directory may contain .xml and .metadata files grouped by pairs and alphabetically classified:" + echo " .hex.as.xml: ANNA-Diameter XML message format." + echo " .metadata: Some decoded message information." + echo " ..." + echo " origin-hosts: Example of content:" + echo " OCS3GPPNodeHostname.ocs3gpprealm.com server" + echo " ggsnNodeHostname.nodeHostRealm.com client" echo - echo " 11.hex.as.xml 15.hex.as.xml 19.hex.as.xml ..." - echo " 11.metadata 15.metadata 19.metadata ..." - echo " 13.hex.as.xml 17.hex.as.xml 21.hex.as.xml ..." - echo " 13.metadata 17.metadata 21.metadata ..." - echo - echo " The order comes from first dot-separated part, which is normally the decoded frame from a pcap file:" - echo - echo " 11.hex.as.xml: would be the ANNA-Diameter decoded message from hexadecimal sniffed one." - echo " 11.metadata: would contain parsed metadata from tshark or another fulfilling this information:" - echo - echo " date=Sat Dec 19 11:50:55 CET 2015" - echo " timestamp=1450522255.205673000" - echo " src=gentraf" - echo " dst=testbed" + echo " The order for the xml and metadata files comes from first dot-separated part (), which is normally the" + echo " decoded frame from a pcap file. The xml files are ANNA-Diameter xml messages and a typical metadata file" + echo " would be something like this:" + echo " date=vie ene 29 15:37:14 CET 2016" + echo " timestamp=1454078234.050988000" + echo " src=192.168.14.42" + echo " dst=192.168.12.40" echo " code=257" - echo " isrequest=1" + echo " isrequest=0" echo " applicationid=0" + echo " originhost=OCSNodeHostname.ocsrealm.com" echo - echo " Presented example has been achieved using the 'example/diameter/pcapDecoder/tsharkDecoder.sh' script" - echo " and then decoding with 'example/diameter/batchConverter' tool with appropiate diameter dictionaries." - echo - echo - echo " test_end_points: space-separated list of possible test end points (hostnames, ip addresses)" - echo - echo " This script will create a basic testcase based on frames timeline (11, 13, 15, etc.), from the desired" - echo " end-point list. For example, from the 'gentraf' point of view, outgoing requests will be translated as" - echo " sendxml operations with corresponding waits for answers and corresponding result codes (normally 2001)" - echo " if those answers are provided (2001 will be assigned as result-code condition if no answer is found)." - echo " CER/A are detected creating a symbolic link to better reference them, and DPR/A are ignored." - echo - echo " The provided test end point list shall match metadata 'src' field in order to filter the valid testcase" - echo " messages. In case of nothing provided (empty string) a list with the possible values from the source" - echo " directory will be shown." + echo " You can use the 'example/diameter/pcapDecoder/tsharkDecoder.sh' script over a pcap file, in order to get" + echo " the hexadecimal buffers and then process them with 'example/diameter/batchConverter' tool to get the xml" + echo " messages and metadata for each pcap diameter frame." echo echo - echo " timeout:" - echo + echo " seconds_timeout:" echo " Timeout in seconds for the testcase. If not provided, no timeout will be programmed for the testcase." echo echo echo echo " The resulting '$TESTCASE_BN' will have all the operations needed to program the test case and will be" - echo " written on same source directory referencing xml files as local-relative ones. Probably, 'change-dir'" + echo " written on same source directory referencing xml files as local-relative ones. Normally, 'change-dir'" echo " operation will be used before programming to ease the procedure without having to copy the stuff into" echo " installed ADML execution directory." echo echo " Connection issues (CER/A), disconnect procedures (DPR/A), database populations and sanity checks are" - echo " not responsability for this script. Such operations shall be externally performed to guarantee that" - echo " programming this testcase is going to be valid in context of test execution. Anyway, CER messages" - echo " will be detected and symbolically linked with the name 'cer..xml'." - echo - echo " PCAPS WITHOUT CER/CEA MESSAGES:" + echo " not responsability for this script. Anyway, CER/A messages detected are symbolically linked with the" + echo " name 'ce..xml', and missing ones are created with a basic template which the user" + echo " could edit and fix with the appropiate values." echo - echo " There is a helper tool to build CER/CEA messages for involved origin hosts. This is useful when the" - echo " pcap file analyzed miss the capabilities exchange messages: touch the file 'create_cer_and_cea' inside" - echo " the source directory, and follow the instructions." _exit } -# $1: xml file -getOriginHost () { - grep " /dev/null @@ -119,12 +105,12 @@ update_testcase () { local sessionid=$(grep Session-Id $xml | cut -d\" -f4) local lines=$(wc -l $TESTCASE_BN | awk '{ print $1 }') - local s_waitfe="test|1|waitfe|$code|$((1-isrequest))" - [ -n "$sessionid" ] && s_waitfe="${s_waitfe}|||${sessionid}" + local s_wait="test|1|$wait_command|$code|$((1-isrequest))" + [ -n "$sessionid" ] && s_wait="${s_wait}|||${sessionid}" if [ $isrequest -eq 1 ] then - echo "test|1|sendxml2e|$xml" >> $TESTCASE_BN + echo "test|1|$send_command|$xml" >> $TESTCASE_BN if [ -n "$resultcode" ] then local hbh="$(grep -o "hop-by-hop-id=\"[0-9]*\"" $xml)" @@ -136,12 +122,12 @@ update_testcase () { _rc=$(getResultCode $ans_xml) [ -n "$_rc" ] && rc=$_rc fi - s_waitfe="${s_waitfe}|${rc}" + s_wait="${s_wait}|${rc}" fi - echo "$s_waitfe" >> $TESTCASE_BN + echo "$s_wait" >> $TESTCASE_BN else - echo "$s_waitfe" >> $TESTCASE_BN - echo "test|1|sendxml2e|$xml|$((lines+1))" >> $TESTCASE_BN + echo "$s_wait" >> $TESTCASE_BN + echo "test|1|$send_command|$xml|$((lines+1))" >> $TESTCASE_BN fi } @@ -159,36 +145,23 @@ cd $SOURCE_DIR ls *.*metadata >/dev/null 2>/dev/null [ $? -ne 0 ] && _exit "Cannot found '*.*metadata' files on '$SOURCE_DIR' !!" -# Show possible end points if nothing provided: -END_POINTS="$2" -if [ -z "$END_POINTS" ] -then - src_eps=( $(grep "^src=" *.*metadata | cut -d= -f2 | sort -u) ) - [ ${#src_eps[@]} -eq 0 ] && _exit "No source end-points detected: metadata must have an 'src=xxx' line." - if [ ${#src_eps[@]} -eq 1 ] - then - END_POINTS=$src_eps - echo "Detected a unique end-point: $END_POINTS" - else - echo "Input a space-separated list for desired end-points to be processed as test side: " - echo " (available source end-points: ${src_eps[*]})" - read END_POINTS - [ -z "$END_POINTS" ] && _exit "Invalid empty input !!" - fi -fi +# Check origin-hosts: +ohs_file=$SOURCE_DIR/origin-hosts +[ ! -f $ohs_file ] && _exit "Can't found the mandatory file '$ohs_file' !!" +awk '{ print $NF }' $ohs_file | egrep -qw 'client|server' +[ ${PIPESTATUS[1]} -ne 0 ] && _exit "Can't found any Origin-Host in '$ohs_file' with a valid ADML node role value (client or server) !!" # Identify useful frames: those which are created at test-bed side (all except frames coming from tested systems): rm -f *.needed -for tag in $END_POINTS +for oh in `awk '{ print $1 }' $ohs_file` do - for metadata in $(grep "^src=$tag$" *.*metadata) + for frame in $(grep -l "^originhost=$oh$" *.*metadata | cut -d\. -f1) do - frame=$(echo $metadata | cut -d\. -f1) touch ${frame}.needed done done ls *.needed >/dev/null 2>/dev/null -[ $? -ne 0 ] && _exit "No frame has been selected within '$SOURCE_DIR' for provided end-points ($END_POINTS) !!" +[ $? -ne 0 ] && _exit "No frame has been selected within '$SOURCE_DIR' for provided origin-hosts file !!" # Messages classification: 0> cers_4_starting @@ -210,6 +183,7 @@ do [ ${#mtd[@]} -ne 1 ] && _exit "There must be one metadata file corresponding to frame '$frame' !!" xml=( `ls ${frame}.*xml 2>/dev/null` ) [ ${#xml[@]} -ne 1 ] && _exit "There must be one xml message file corresponding to '$mtd' (frame '$frame') !!" + originHost=$(grep ^originhost= ${frame}.metadata | cut -d= -f2-) # Ignore keep alives: grep -q "^code=280$" $mtd @@ -223,7 +197,6 @@ do if [ $? -eq 0 ] then echo $frame >> cers_4_starting - originHost=$(getOriginHost $xml) [ -z "$originHost" ] && _exit "Missing Origin-Host (frame $frame, CER message) !!" echo "$originHost" >> cers_4_starting_origin_hosts ln -sf $xml cer.${originHost}.xml @@ -231,14 +204,17 @@ do fi # Other requests: echo $frame >> requests_4_sending - update_testcase $mtd $xml check_result_code + + # client or server: + adml_type=$(grep -w "$originHost" $ohs_file | awk '{ print $NF }') + update_testcase $mtd $xml check_result_code $adml_type else # CEA's: grep -q "^code=257$" $mtd if [ $? -eq 0 ] then echo $frame >> ceas_4_establishing - originHost=$(getOriginHost $xml) + originHost=$(grep ^originhost= ${frame}.metadata | cut -d= -f2-) [ -z "$originHost" ] && _exit "Missing Origin-Host (frame $frame, CEA message) !!" echo "$originHost" >> ceas_4_establishing_origin_hosts ln -sf $xml cea.${originHost}.xml @@ -246,7 +222,10 @@ do fi # Other answers: echo $frame >> answers_4_programming - update_testcase $mtd $xml + + # client or server: + adml_type=$(grep -w "$originHost" $ohs_file | awk '{ print $NF }') + update_testcase $mtd $xml "" $adml_type fi done rm -f *.needed @@ -258,7 +237,7 @@ hbh_ini= hbh_fin= # involved frames: involved_xmls= -grep sendxml $TESTCASE_BN > .involved_frames +grep "|sendxml" $TESTCASE_BN > .involved_frames n_involved=`wc -l .involved_frames | awk '{ print $1 }'` count=1 for frame in `cat .involved_frames | cut -d\| -f4 | cut -d\. -f1` @@ -275,21 +254,19 @@ do count=$((count+1)) done -if [ -f $SOURCE_DIR/create_cer_and_cea ] -then - # PCAPs without CER messages ... - if [ ! -s cers_4_starting_origin_hosts ] +# Involved origin hosts: +grep " .involved_origin_hosts +grep -w -f .involved_origin_hosts $ohs_file > .involved_origin_hosts_with_info + +# Missing CERs: +for oh in $(awk -v input=client '{if ($2 == input) print $1;}' .involved_origin_hosts_with_info) +do + cer=$SOURCE_DIR/cer.${oh}.xml + if [ ! -f $cer ] then - if [ $n_involved -ne 0 ] - then - grep " cers_4_starting_origin_hosts - for oh in `cat cers_4_starting_origin_hosts` - do - cer_file=cer.${oh}.xml - touch $cer_file - echo "WARNING: perhaps you should configure '$cer_file' because it is missing in the pcap file provided." - echo " Currently it contains a basic template but you must fill/fix the unknowns '$WHAT'." - cat << EOF > $cer_file + echo $oh >> cers_4_starting_origin_hosts + echo "Missing CER: `basename $cer` (a basic template has been created, please edit & fix the unknowns)" + cat << EOF > $cer @@ -301,25 +278,18 @@ then EOF - done - #else - # echo "WARNING: No 'sendxml' primitives on testcase ??" - fi fi +done - # PCAPs without CEA messages ... - if [ ! -s ceas_4_establishing_origin_hosts ] +# Missing CEAs: +for oh in $(awk -v input=server '{if ($2 == input) print $1;}' .involved_origin_hosts_with_info) +do + cea=$SOURCE_DIR/cea.${oh}.xml + if [ ! -f $cea ] then - if [ $n_involved -ne 0 ] - then - grep " ceas_4_establishing_origin_hosts - for oh in `cat ceas_4_establishing_origin_hosts` - do - cea_file=cea.${oh}.xml - touch $cea_file - echo "WARNING: perhaps you should configure '$cea_file' because it is missing in the pcap file provided." - echo " Currently it contains a basic template but you must fill/fix the unknowns '$WHAT'." - cat << EOF > $cea_file + echo $oh >> ceas_4_establishing_origin_hosts + echo "Missing CEA: `basename $cea` (a basic template has been created, please edit & fix the unknowns)" + cat << EOF > $cea @@ -342,12 +312,10 @@ EOF EOF - done - #else - # echo "WARNING: No 'sendxml' primitives on testcase ??" - fi fi -fi +done + +rm -f .involved_origin_hosts .involved_origin_hosts_with_info exit 0 diff --git a/example/diameter/pcapDecoder/tsharkDecoder.sh b/example/diameter/pcapDecoder/tsharkDecoder.sh index deca8eb..c290e17 100755 --- a/example/diameter/pcapDecoder/tsharkDecoder.sh +++ b/example/diameter/pcapDecoder/tsharkDecoder.sh @@ -42,6 +42,7 @@ # VARIABLES # ############# tmpdir=$(mktemp -d) +TSHARK=tshark ############# # FUNCTIONS # @@ -64,9 +65,9 @@ usage() { echo echo " pcap: pcap formatted file to be processed." echo - echo " The utility, dumps the extracted hexadecimal content" - echo " and useful information as timestamps, source and" - echo " destination:" + echo " The utility, dumps the extracted hexadecimal content and useful information" + echo " (timestamps, source, destination, etc.) within a metadata file:" + echo echo " /.hex" echo " /.metadata" echo @@ -149,12 +150,12 @@ echo parse_arguments "$@" # Tshark available: -which tshark >/dev/null +which $TSHARK >/dev/null 2>/dev/null [ $? -ne 0 ] && _exit "Missing 'tshark' tool !!" # Get the frames with diameter content (take care about '-2' two-pass option and don't add it, because we need to get reassembled parts in their corresponding frames): # Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): see https://www.wireshark.org/docs/dfref/d/diameter.html -FIELDS_DIAMETER="-e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.length" +FIELDS_DIAMETER="-e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.Origin-Host -e diameter.length" FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host $FIELDS_DIAMETER -e tcp.len -e frame.protocols -e tcp.segment" # Disect selectors for non-standard diameter ports: for port in $OTHER_PORTS @@ -163,19 +164,19 @@ do DISECT_SELECTORS="$DISECT_SELECTORS -d tcp.port=$port,diameter" done -tshark -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames +$TSHARK -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames # Example output: -# /length\ -# frame timestamp src dst code R App-ID HopByHop EndToEnd Session-Id DIAM TCP protocol segments -# 1 2 3 4 5 6 7 8 9 10 11 12 13 14 -# 1|1427215933.697904000|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxxxxxx|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp| -# 3|1427215934.449523000|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxxxxxx|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp| +# /length\ +# frame timestamp src dst code R App-ID HopByHop EndToEnd Session-Id Origin-Host DIAM TCP protocol segments +# 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +# 1|1427215933.697904000|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxxxxxx|xxxxxxxxxxxxx|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp| +# 3|1427215934.449523000|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxxxxxx|xxxxxxxxxxxxx|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp| # 5|1427215934.456160000|gt_traf|vcbavipt||||||||1400|eth:ip:tcp:diameter| -# 6|1427215934.456204000|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxxxxxx|1972|572|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6 -# 8|1427215935.123559000|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxxxxxx|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp| +# 6|1427215934.456204000|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxxxxxx|xxxxxxxxxxxxx|1972|572|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6 +# 8|1427215935.123559000|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxxxxxx|xxxxxxxxxxxxx|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp| all_frames=( $(cat $tmpdir/diameter_frames | cut -d\| -f1) ) -needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f14) ) -main_frames=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($12 != "") print $1 }') ) +needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f15) ) +main_frames=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($13 != "") print $1 }') ) # Reassemble procedure (using frame 1 as example): # (for non segmented frames, it is enough with tcp or diameter length within the frame content itself) @@ -190,7 +191,7 @@ for frame in ${all_frames[@]}; do frame_info=$(grep "^${frame}|" $tmpdir/diameter_frames) # Get the diameter part: - tcp_len=$(echo $frame_info | cut -d\| -f12) + tcp_len=$(echo $frame_info | cut -d\| -f13) frm_len=$(wc -c $tmpdir/block.$frame | awk '{ print $1 }') cut_len=$((frm_len-2*tcp_len)) cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex @@ -205,6 +206,7 @@ for frame in ${all_frames[@]}; do isreq=$(echo $frame_info | cut -d\| -f6) appid=$(echo $frame_info | cut -d\| -f7) sid=$(echo $frame_info | cut -d\| -f10) + oh=$(echo $frame_info | cut -d\| -f11) #hbh=$(echo $frame_info | cut -d\| -f8) #e2e=$(echo $frame_info | cut -d\| -f9) # HBH and ETE To decimal: @@ -218,6 +220,7 @@ for frame in ${all_frames[@]}; do echo "isrequest=$isreq" >> $RESULTS_DIR/$frame.metadata echo "applicationid=$appid" >> $RESULTS_DIR/$frame.metadata [ -n "$sid" ] && echo "sessionid=$sid" >> $RESULTS_DIR/$frame.metadata + echo "originhost=$oh" >> $RESULTS_DIR/$frame.metadata #echo "hopbyhop=$hbh" >> $RESULTS_DIR/$frame.metadata #echo "endtoend=$e2e" >> $RESULTS_DIR/$frame.metadata @@ -236,7 +239,7 @@ done # Delete superfluous metadata: echo "Deleting superfluous buffers & metadata ..." -segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($11 == "") print $1 }') ) +segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($12 == "") print $1 }') ) for s in ${segments[@]}; do rm $RESULTS_DIR/$s.*; done # Detecting Session-Id values: @@ -244,10 +247,19 @@ grep ^sessionid= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > if [ -s $RESULTS_DIR/session-ids ] then count=0 - while read -r line; do count=$((count+1)) ; echo "Detected Session-Id $count: $line"; done < $RESULTS_DIR/session-ids + while read -r line; do count=$((count+1)) ; echo "Detected Session-Id $count: $line"; done < $RESULTS_DIR/session-ids rm $RESULTS_DIR/session-ids fi +# Detecting Origin-Host values: +grep ^originhost= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/origin-hosts +if [ -s $RESULTS_DIR/origin-hosts ] +then + count=0 + while read -r line; do count=$((count+1)) ; echo "Detected Origin-Host $count: $line"; done < $RESULTS_DIR/origin-hosts + #rm $RESULTS_DIR/origin-hosts +fi + # Purge frames with Session-Id not wanted: if [ -n "$SIDS_FILE" ] then -- 2.20.1