From 7bf36f6cec41494071f07699184d8230ccd8cb41 Mon Sep 17 00:00:00 2001 From: Eduardo Ramos Testillano Date: Sat, 28 Mar 2015 04:40:14 +0100 Subject: [PATCH] Improvements & fixes Improved deployment of examples. Default deployment dir depending on type of example. Added a new script to transform a raw pcap into diameter hex content & metadata, ready for launcher encoder. Launcher accepts now hexadecimal content with colons (as rawshark output) --- example/diameter/launcher/DEPLOY.sh | 27 ++-- .../{resources => deployments}/advanced/1.xml | 0 .../advanced/1xml-23.txt | 0 .../advanced/README | 2 + .../advanced/answerXml.sh | 0 .../advanced/args.txt | 0 .../advanced/batch.sh | 0 .../advanced/burst.sh | 0 .../advanced/clientSocketManager.sh | 0 .../advanced/code.sh | 0 .../advanced/collect.sh | 0 .../advanced/data-initial.msk | 0 .../advanced/data-termination.msk | 0 .../advanced/data-update1.msk | 0 .../advanced/data-update2.msk | 0 .../advanced/data.sh | 0 .../advanced/decode.sh | 0 .../advanced/diameterServerSessions.sh | 0 .../advanced/help.sh | 0 .../advanced/loadXml.sh | 0 .../advanced/pre-start.sh | 0 .../advanced/sendXml.sh | 0 .../advanced/sms.msk | 0 .../advanced/sms.sh | 0 .../{resources => deployments}/basic/README | 2 + .../basic/configure.sh | 0 .../basic/hex_examples/aar-bad.hex | 0 .../basic/hex_examples/aar.hex | 0 .../basic/hex_examples/aar2-bad.hex | 0 .../basic/hex_examples/aar3-bad.hex | 0 .../basic/hex_examples/readme.txt | 0 .../basic/hex_examples/tspCCA.hex | 0 .../basic/hex_examples/tspCCR.hex | 0 .../basic/operation.sh | 0 .../basic/xml_examples/aar.xml | 0 .../basic/xml_examples/aar2.xml | 0 .../basic/xml_examples/ccr.xml | 0 .../ft-client/README | 2 + .../ft-client/args.txt | 0 .../ft-client/hex_examples | 0 .../ft-client/operation.sh | 0 .../ft-client/pre-start.sh | 0 .../ft-client/tests/experiment1/case_1.sh | 0 .../ft-client/tests/experiment1/common.sh | 0 .../ProtocolErrors/BadAARtoServer/aar-bad.hex | 0 .../BadAARtoServer/aar2-bad.hex | 0 .../BadAARtoServer/aar3-bad.hex | 0 .../ProtocolErrors/BadAARtoServer/case_1.tc | 0 .../ProtocolErrors/BadAARtoServer/case_2.tc | 0 .../ProtocolErrors/BadAARtoServer/case_3.tc | 0 .../ft-client/tests/experiment2/go.sh | 0 .../ft-client/xml_examples | 0 example/diameter/launcher/main.cpp | 8 ++ .../launcher/resources/pcap2diameterHex.sh | 135 ++++++++++++++++++ 54 files changed, 165 insertions(+), 11 deletions(-) rename example/diameter/launcher/{resources => deployments}/advanced/1.xml (100%) rename example/diameter/launcher/{resources => deployments}/advanced/1xml-23.txt (100%) rename example/diameter/launcher/{resources => deployments}/advanced/README (93%) rename example/diameter/launcher/{resources => deployments}/advanced/answerXml.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/args.txt (100%) rename example/diameter/launcher/{resources => deployments}/advanced/batch.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/burst.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/clientSocketManager.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/code.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/collect.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/data-initial.msk (100%) rename example/diameter/launcher/{resources => deployments}/advanced/data-termination.msk (100%) rename example/diameter/launcher/{resources => deployments}/advanced/data-update1.msk (100%) rename example/diameter/launcher/{resources => deployments}/advanced/data-update2.msk (100%) rename example/diameter/launcher/{resources => deployments}/advanced/data.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/decode.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/diameterServerSessions.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/help.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/loadXml.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/pre-start.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/sendXml.sh (100%) rename example/diameter/launcher/{resources => deployments}/advanced/sms.msk (100%) rename example/diameter/launcher/{resources => deployments}/advanced/sms.sh (100%) rename example/diameter/launcher/{resources => deployments}/basic/README (95%) rename example/diameter/launcher/{resources => deployments}/basic/configure.sh (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/aar-bad.hex (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/aar.hex (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/aar2-bad.hex (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/aar3-bad.hex (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/readme.txt (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/tspCCA.hex (100%) rename example/diameter/launcher/{resources => deployments}/basic/hex_examples/tspCCR.hex (100%) rename example/diameter/launcher/{resources => deployments}/basic/operation.sh (100%) rename example/diameter/launcher/{resources => deployments}/basic/xml_examples/aar.xml (100%) rename example/diameter/launcher/{resources => deployments}/basic/xml_examples/aar2.xml (100%) rename example/diameter/launcher/{resources => deployments}/basic/xml_examples/ccr.xml (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/README (95%) rename example/diameter/launcher/{resources => deployments}/ft-client/args.txt (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/hex_examples (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/operation.sh (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/pre-start.sh (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment1/case_1.sh (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment1/common.sh (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/ProtocolErrors/BadAARtoServer/aar-bad.hex (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/ProtocolErrors/BadAARtoServer/aar2-bad.hex (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/ProtocolErrors/BadAARtoServer/aar3-bad.hex (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/ProtocolErrors/BadAARtoServer/case_1.tc (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/ProtocolErrors/BadAARtoServer/case_2.tc (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/ProtocolErrors/BadAARtoServer/case_3.tc (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/tests/experiment2/go.sh (100%) rename example/diameter/launcher/{resources => deployments}/ft-client/xml_examples (100%) create mode 100755 example/diameter/launcher/resources/pcap2diameterHex.sh diff --git a/example/diameter/launcher/DEPLOY.sh b/example/diameter/launcher/DEPLOY.sh index eaec5b3..76ccd8b 100755 --- a/example/diameter/launcher/DEPLOY.sh +++ b/example/diameter/launcher/DEPLOY.sh @@ -8,7 +8,6 @@ MSGDTD=../../../include/anna/diameter/codec/message.dtd DCTDTD=../../../include/anna/diameter/stack/dictionary.dtd BASE_PROT=commands_baseProtocol.xml EXEC=./debug/example_diameter_launcher -DPATH_dflt=$HOME/ADL ############# # FUNCTIONS # @@ -78,14 +77,9 @@ echo "---------------------------------------" echo "Anna Diameter Launcher (ADL) deployment" echo "---------------------------------------" echo -echo "Input deployment path [$DPATH_dflt]:" -read DPATH -[ "$DPATH" = "" ] && DPATH=$DPATH_dflt -[ -d $DPATH ] && _exit "The path '$DPATH' already exists. Remove it before continue..." - echo "Basic checkings ..." [ ! -f $EXEC ] && _exit "Anna Diameter Launcher (ADL) is not linked. Execute 'scons' for 'anna' suite." -[ ! -d $SETUPS_DIR ] && _exit "Diameter stacks not found ($SETUPS_DIR). Perhaps you executed this script out of its parent path." +[ ! -d $SETUPS_DIR ] && _exit "Diameter stacks not found ($SETUPS_DIR)." echo echo "Deploy one of these versions:" @@ -99,15 +93,18 @@ read option [ "$option" = "" ] && option=b case $option in a) - RESOURCES_DIR=resources/advanced + DEPLOYMENTS_DIR=deployments/advanced + DPATH_dflt=$HOME/ADL-advanced ;; b) - RESOURCES_DIR=resources/basic + DEPLOYMENTS_DIR=deployments/basic + DPATH_dflt=$HOME/ADL-basic ;; f) - RESOURCES_DIR=resources/ft-client + DEPLOYMENTS_DIR=deployments/ft-client + DPATH_dflt=$HOME/ADL-ft-client ;; *) @@ -115,13 +112,21 @@ case $option in ;; esac +echo +echo "Input deployment path [$DPATH_dflt]:" +read DPATH +[ "$DPATH" = "" ] && DPATH=$DPATH_dflt +[ -d $DPATH ] && _exit "The path '$DPATH' already exists. Remove it before continue..." + echo "Copying ..." mkdir -p $DPATH mkdir -p $DPATH/stacks mkdir -p $DPATH/DTDs mkdir -p $DPATH/counters +mkdir -p $DPATH/resources cp $EXEC $DPATH/ADL-launcher -cp -rL $RESOURCES_DIR/* $DPATH +cp -rL $DEPLOYMENTS_DIR/* $DPATH +cp resources/* $DPATH/resources cp $SETUPS_DIR/*xml $DPATH/stacks cp $SETUPS_DIR/*sh $DPATH/stacks cp $SETUPS_DIR/readme.txt $DPATH/stacks diff --git a/example/diameter/launcher/resources/advanced/1.xml b/example/diameter/launcher/deployments/advanced/1.xml similarity index 100% rename from example/diameter/launcher/resources/advanced/1.xml rename to example/diameter/launcher/deployments/advanced/1.xml diff --git a/example/diameter/launcher/resources/advanced/1xml-23.txt b/example/diameter/launcher/deployments/advanced/1xml-23.txt similarity index 100% rename from example/diameter/launcher/resources/advanced/1xml-23.txt rename to example/diameter/launcher/deployments/advanced/1xml-23.txt diff --git a/example/diameter/launcher/resources/advanced/README b/example/diameter/launcher/deployments/advanced/README similarity index 93% rename from example/diameter/launcher/resources/advanced/README rename to example/diameter/launcher/deployments/advanced/README index 52393f8..9e0b3b6 100644 --- a/example/diameter/launcher/resources/advanced/README +++ b/example/diameter/launcher/deployments/advanced/README @@ -8,6 +8,8 @@ Available different setups by mean files named as '.dictionary__> buffer; std::string hexString(buffer, strlen(buffer)); + // Allow colon separator in hex string: we have to remove them before processing with 'fromHexString': + hexString.erase(std::remove(hexString.begin(), hexString.end(), ':'), hexString.end()); + LOGDEBUG( + std::string msg = "Hex string (remove colons if exists): "; + msg += hexString; + anna::Logger::debug(msg, ANNA_FILE_LOCATION); + ); + anna::functions::fromHexString(hexString, db); // Close file infile.close(); diff --git a/example/diameter/launcher/resources/pcap2diameterHex.sh b/example/diameter/launcher/resources/pcap2diameterHex.sh new file mode 100755 index 0000000..45dd820 --- /dev/null +++ b/example/diameter/launcher/resources/pcap2diameterHex.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +############# +# VARIABLES # +############# +tmpdir=$(mktemp -d) + +############# +# FUNCTIONS # +############# + +usage () { + echo "Usage: $0 [results_dir]" + echo + echo " pcap_file: pcap formatted file to be processed." + echo " results_dir: directory where results are stored." + echo " By default, pcap file dirname is used." + echo + echo " The utility, dumps the extracted hexadecimal content" + echo " and useful information as timestamps, source and" + echo " destination:" + echo " /.hex" + echo " /.metadata" + echo + _exit +} + +_exit () { + echo + echo -e $1 + echo + + # Cleanup + #rm -rf $tmpdir + + rc=1 + [ -n "$2" ] && rc=$2 + exit $rc +} + +# $1: string; $2: substring +get_str_pos() { + local result=-1 + local str=$1 + local substr="${1%%$2*}" + local pos=${#substr} + local size=${#str} + [ $size -ne $pos ] && result=$pos + echo $result +} + +############# +# EXECUTION # +############# + +echo +echo "============================================" +echo "Diameter buffer extractor from PCAP raw file" +echo "============================================" +echo + +# Usage: +[ "$1" = "" ] && usage + +# Pcap file: +PCAP_FILE=$1 +[ ! -f $PCAP_FILE ] && _exit "Cannot found provided pcap file '$1' !!" + +# Optional result dir: +RESULTS_DIR=`dirname $PCAP_FILE` +[ "$2" != "" ] && RESULTS_DIR=$2 +[ ! -d $RESULTS_DIR ] && _exit "The results directory '$RESULTS_DIR' must exists !!" + +# Get the frames with diameter content (take care about '-2' two-pass option and don't add it, because we need to get reassembled parts in their corresponding frames): +# Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): +FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host -e tcp.len -e diameter.length -e frame.protocols -e tcp.segment" +tshark -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames +# Example output: +# /length\ +# frame timestamp src dst TCP DIAM protocol segments +# 1|1427215933.697904000|gt_traf|vcbavipt|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp| +# 3|1427215934.449523000|vcbavipt|gt_traf|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp| +# 5|1427215934.456160000|gt_traf|vcbavipt|1400||eth:ip:tcp:diameter| +# 6|1427215934.456204000|gt_traf|vcbavipt|572|1972|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6 +# 8|1427215935.123559000|vcbavipt|gt_traf|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp| +all_frames=( $(cat $tmpdir/diameter_frames | cut -d\| -f1) ) +needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f8) ) +main_frames=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($6 != "") print $1 }') ) + +# Reassemble procedure (using frame 1 as example): +# (for non segmented frames, it is enough with tcp or diameter length within the frame content itself) +# 1) Get the TCP length: 432 bytes. 432*2 = 864 characters per byte in hexadecimal string format +# 2) Get the frame length: `wc -c $tmpdir/block.$frame` => 997 +# 3) Get 864 from the tail: `cat $tmpdir/block.$frame | cut -c133 + +# Dump the hex blocks for all the diameter frames: +cat $PCAP_FILE | rawshark -s -r - -d proto:diameter -F data 2>/dev/null > $tmpdir/all_hex_data +for frame in ${all_frames[@]}; do + grep "^$frame " $tmpdir/all_hex_data | cut -d\" -f2 | sed 's/://g' > $tmpdir/block.$frame + frame_info=$(grep "^${frame}|" $tmpdir/diameter_frames) + + # Get the diameter part: + tcp_len=$(echo $frame_info | cut -d\| -f5) + frm_len=$(wc -c $tmpdir/block.$frame | awk '{ print $1 }') + cut_len=$((frm_len-2*tcp_len)) + cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex + echo -n "Created $RESULTS_DIR/$frame.hex" + + # Metadata: + ts=$(echo $frame_info | cut -d\| -f2) + date=$(date -d @$ts) + src=$(echo $frame_info | cut -d\| -f3) + dst=$(echo $frame_info | cut -d\| -f4) + echo -e "timestamp=$ts\ndate=$date\nsrc=$src\ndst=$dst" > $RESULTS_DIR/$frame.metadata + echo " and $RESULTS_DIR/$frame.metadata" +done + +# Join frames which need to be reassembled: +for group in ${needs_join[@]}; do + echo "Grouping frames $group ..." + group_array=( $(echo $group | sed 's/,/ /g') ) + for frame in ${group_array[@]}; do + cat $RESULTS_DIR/$frame.hex >> $tmpdir/diam.$group + done + cat $tmpdir/diam.$group | tr -d '\n' > $RESULTS_DIR/$frame.hex +done + +# Delete superfluous metadata: +echo "Deleting superfluous metadata ..." +segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($6 == "") print $1 }') ) +for s in ${segments[@]}; do rm $RESULTS_DIR/$s.metadata; done + + +_exit "Done!" 0 + -- 2.20.1