X-Git-Url: https://git.teslayout.com/public/public/public/?a=blobdiff_plain;f=example%2Fdiameter%2FpcapDecoder%2FtsharkDecoder.sh;h=93b29d42e1b160f2a7276628d277edec9685a72b;hb=eb49b785a7e32e6259c7689f3861beed9b5a1a85;hp=90c33849fa54405e71a97268bdb01a789a6673b3;hpb=1dc9526e0683b753715f9a725b066b06ea1b439e;p=anna.git diff --git a/example/diameter/pcapDecoder/tsharkDecoder.sh b/example/diameter/pcapDecoder/tsharkDecoder.sh index 90c3384..93b29d4 100755 --- a/example/diameter/pcapDecoder/tsharkDecoder.sh +++ b/example/diameter/pcapDecoder/tsharkDecoder.sh @@ -37,11 +37,14 @@ # Decoder version using tshark tool +# +# RESTRICTIONS: only TCP datagrams. Allow split diameter messages between frames, but not, several messages into single datagram. ############# # VARIABLES # ############# tmpdir=$(mktemp -d) +TSHARK=tshark ############# # FUNCTIONS # @@ -49,21 +52,24 @@ tmpdir=$(mktemp -d) usage() { echo - echo "Usage: $0 [-h|--help] [-o|--other-ports] [-d|--results-dir] " + echo "Usage: $0 [-h|--help] [-o|--other-ports ] [-d|--results-dir ] [-s|--sids ] " + echo + echo " -h|--help: this usage help." + echo " -o|--other-ports : space-separated list of ports which frames" + echo " will be decoded as diameter protocol although" + echo " not being standard. For example, we could use" + echo " \"13868\" to disect the Ericsson Sy variant." + echo " -d|--results-dir : directory where results are stored." + echo " By default, pcap dirname." + echo " -s|--sids : file containing a list of Session-Id values" + echo " (one per line) to be taken into account." + echo " If missing, all the frames will be extracted." echo - echo " -h|--help: this usage help." - echo " -o|--other-ports: space-separated list of ports which frames" - echo " will be decoded as diameter protocol although" - echo " not being standard. For example, we could use" - echo " \"13868\" to disect the Ericsson Sy variant." - echo " -d|--results-dir: directory where results are stored." - echo " By default, pcap dirname." + echo " pcap: pcap formatted file to be processed." echo - echo " pcap: pcap formatted file to be processed." + echo " The utility, dumps the extracted hexadecimal content and useful information" + echo " (timestamps, source, destination, etc.) within a metadata file:" echo - echo " The utility, dumps the extracted hexadecimal content" - echo " and useful information as timestamps, source and" - echo " destination:" echo " /.hex" echo " /.metadata" echo @@ -73,6 +79,7 @@ usage() { parse_arguments() { OTHER_PORTS= RESULTS_DIR= + SIDS_FILE= PCAP_FILE= while [ $# -gt 0 ]; do @@ -92,6 +99,11 @@ parse_arguments() { shift ;; + -s|--sids) + SIDS_FILE=$2 + shift + ;; + *) first=$(echo $1 | cut -c1) [ "$first" = "-" ] && _exit "Unsupported script option: $1. Type '$SCR_BN -h' (or --help) to print the available options." @@ -105,6 +117,10 @@ parse_arguments() { [ ! -f "$PCAP_FILE" ] && _exit "Cannot found provided pcap file '$PCAP_FILE' !!" [ -z "$RESULTS_DIR" ] && RESULTS_DIR=`dirname $PCAP_FILE` [ ! -d $RESULTS_DIR ] && _exit "The results directory '$RESULTS_DIR' must exists !!" + if [ -n "$SIDS_FILE" ] + then + [ ! -f $SIDS_FILE ] && _exit "The Session-Id list file '$SIDS_FILE' provided, does not exist !!" + fi } _exit () { @@ -136,13 +152,12 @@ echo parse_arguments "$@" # Tshark available: -which tshark >/dev/null +which $TSHARK >/dev/null 2>/dev/null [ $? -ne 0 ] && _exit "Missing 'tshark' tool !!" # Get the frames with diameter content (take care about '-2' two-pass option and don't add it, because we need to get reassembled parts in their corresponding frames): -# Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): -FIELDS_DIAMETER="-e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.length" -FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host $FIELDS_DIAMETER -e tcp.len -e frame.protocols -e tcp.segment" +# Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): see https://www.wireshark.org/docs/dfref/d/diameter.html +FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host -e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.Origin-Host -e diameter.Subscription-Id-Data -e diameter.Subscription-Id-Type -e frame.len -e tcp.len -e diameter.length -e frame.protocols -e tcp.segment -e sctp.chunk_length" # Disect selectors for non-standard diameter ports: for port in $OTHER_PORTS do @@ -150,18 +165,38 @@ do DISECT_SELECTORS="$DISECT_SELECTORS -d tcp.port=$port,diameter" done -tshark -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames +$TSHARK -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null > $tmpdir/all_frames +grep -i diameter $tmpdir/all_frames > $tmpdir/diameter_frames +#cat $tmpdir/all_frames + # Example output: -# /length\ -# frame timestamp src dst code R App-ID HopByHop EndToEnd DIAM TCP protocol segments -# 1|1427215933.697904000|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp| -# 3|1427215934.449523000|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp| -# 5|1427215934.456160000|gt_traf|vcbavipt|||||||1400|eth:ip:tcp:diameter| -# 6|1427215934.456204000|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|1972|572|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6 -# 8|1427215935.123559000|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp| -all_frames=( $(cat $tmpdir/diameter_frames | cut -d\| -f1) ) -needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f13) ) -main_frames=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($11 != "") print $1 }') ) +# / lengths \ +# frm timestamp src dst code R App-ID HopByHop EndToEnd Sid OHost Subs SubsType FRM TCP DIA protocol segments sctp chunk length +# 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +# 1|tt.tt|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp| +# 3|tt.tt|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp| +# 5|tt.tt|gt_traf|vcbavipt||||||ffff|1400||eth:ip:tcp:diameter| +# 6|tt.tt|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|572|1972|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6 +# 8|tt.tt|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp| +cat $tmpdir/diameter_frames | cut -d\| -f1 > $tmpdir/frames_num +all_frames=( $(cat $tmpdir/frames_num) ) + +# Segments groups (TCP or SCTP): +segmentsGroups= + +# The case of SCTP is precalculated: +tshark -nr $PCAP_FILE $DISECT_SELECTORS -Tfields -e frame.number -e sctp.chunk_flags 2>/dev/null | awk '{ if ( $2 !~ "0x03" && $2 != "" ) print $1 }' > $tmpdir/extra_frames_num +if [ -s $tmpdir/extra_frames_num ] +then + # Take into account these frames in all_frames array: + + cat $tmpdir/extra_frames_num >> $tmpdir/frames_num + all_frames=( $(cat $tmpdir/frames_num | sort -un) ) + + # Prepare as groups (1,2 7,8 ...) + segmentsGroups=$(tshark -nr $PCAP_FILE $DISECT_SELECTORS -Tfields -e frame.number -e sctp.chunk_flags 2>/dev/null | awk '{ if ( $2 !~ "0x03" && $2 != "" ) print $1; else print "\n" }' | uniq | tr '\n' ',' | sed 's/,,/ /g') +fi + # Reassemble procedure (using frame 1 as example): # (for non segmented frames, it is enough with tcp or diameter length within the frame content itself) @@ -175,11 +210,53 @@ for frame in ${all_frames[@]}; do grep "^$frame " $tmpdir/all_hex_data | cut -d\" -f2 | sed 's/://g' > $tmpdir/block.$frame frame_info=$(grep "^${frame}|" $tmpdir/diameter_frames) + ########################################################################################## # Get the diameter part: - tcp_len=$(echo $frame_info | cut -d\| -f11) - frm_len=$(wc -c $tmpdir/block.$frame | awk '{ print $1 }') - cut_len=$((frm_len-2*tcp_len)) - cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex + frm_len=$(echo $frame_info | cut -d\| -f14) + tcp_len=$(echo $frame_info | cut -d\| -f15) + dia_len=$(echo $frame_info | cut -d\| -f16) + protocol=$(echo $frame_info | cut -d\| -f17) + + # TCP and SCTP supported: + transport=$(echo $protocol | grep -ow tcp) + [ -z "$transport" ] && transport=$(echo $protocol | grep -ow sctp) + # in case of segmented SCTP frames, protocol won't be sctp ...: + if [ -s "$tmpdir/extra_frames_num" -a -z "$transport" ] + then + grep -qw ^${frame} $tmpdir/extra_frames_num + if [ $? -eq 0 ] + then + transport=sctp + frame_info=$(grep "^${frame}|" $tmpdir/all_frames) + fi + fi + + case $transport in + sctp) + chunk_length=$(echo $frame_info | cut -d\| -f19) + dia_partial_len=$((chunk_length-16)) + if [ "$dia_len" != "$dia_partial_len" ] + then + #echo "Segmented SCTP frame $frame; taking partial diameter length = $dia_partial_len" + dia_len=$dia_partial_len + fi + cut_len=$((2*dia_len)) + cat $tmpdir/block.$frame | rev | cut -c-${cut_len} | rev > $RESULTS_DIR/$frame.hex + ;; + + tcp) + cut_len=$((2*(frm_len-tcp_len) + 1)) + cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex + cp $tmpdir/block.$frame /tmp/block + segments=$(echo $frame_info | cut -d\| -f18) + segmentsGroups="$segmentsGroups $segments" + ;; + + *) _exit "Only TCP and SCTP transports supported !!" + ;; + esac + ########################################################################################## + echo -n "Created $RESULTS_DIR/$frame.hex" # Metadata: @@ -190,11 +267,19 @@ for frame in ${all_frames[@]}; do code=$(echo $frame_info | cut -d\| -f5) isreq=$(echo $frame_info | cut -d\| -f6) appid=$(echo $frame_info | cut -d\| -f7) - hbh=$(echo $frame_info | cut -d\| -f8) - e2e=$(echo $frame_info | cut -d\| -f9) - # To decimal: - hbh=$(printf "%d\n" $hbh) - e2e=$(printf "%d\n" $e2e) + sid=$(echo $frame_info | cut -d\| -f10) + oh=$(echo $frame_info | cut -d\| -f11) + subscriber=$(echo $frame_info | cut -d\| -f12) + subscribertype=$(echo $frame_info | cut -d\| -f13) + [ "$subscribertype" = "0" ] && subscribertype=msisdn + [ "$subscribertype" = "1" ] && subscribertype=imsi + + + #hbh=$(echo $frame_info | cut -d\| -f8) + #e2e=$(echo $frame_info | cut -d\| -f9) + # HBH and ETE To decimal: + #hbh=$(printf "%d\n" $hbh) + #e2e=$(printf "%d\n" $e2e) echo "date=$date" > $RESULTS_DIR/$frame.metadata echo "timestamp=$ts" >> $RESULTS_DIR/$frame.metadata echo "src=$src" >> $RESULTS_DIR/$frame.metadata @@ -202,28 +287,71 @@ for frame in ${all_frames[@]}; do echo "code=$code" >> $RESULTS_DIR/$frame.metadata echo "isrequest=$isreq" >> $RESULTS_DIR/$frame.metadata echo "applicationid=$appid" >> $RESULTS_DIR/$frame.metadata - #echo "sequence=${hbh}.${e2e}" >> $RESULTS_DIR/$frame.metadata -# echo "hopbyhop=$hbh" >> $RESULTS_DIR/$frame.metadata -# echo "endtoend=$e2e" >> $RESULTS_DIR/$frame.metadata + [ -n "$sid" ] && echo "sessionid=$sid" >> $RESULTS_DIR/$frame.metadata + echo "originhost=$oh" >> $RESULTS_DIR/$frame.metadata + if [ -n "$subscriber" ] + then + echo "subscriber=$subscriber" >> $RESULTS_DIR/$frame.metadata + echo "subscribertype=$subscribertype" >> $RESULTS_DIR/$frame.metadata + fi + #echo "hopbyhop=$hbh" >> $RESULTS_DIR/$frame.metadata + #echo "endtoend=$e2e" >> $RESULTS_DIR/$frame.metadata echo " and $RESULTS_DIR/$frame.metadata" done # Join frames which need to be reassembled: -for group in ${needs_join[@]}; do - echo "Grouping frames $group ..." +to_delete= +for group in $segmentsGroups +do group_array=( $(echo $group | sed 's/,/ /g') ) + echo "Grouping frames ${group_array[*]} ..." for frame in ${group_array[@]}; do cat $RESULTS_DIR/$frame.hex >> $tmpdir/diam.$group done cat $tmpdir/diam.$group | tr -d '\n' > $RESULTS_DIR/$frame.hex + # Delete all frames except last one in the group: + to_delete="$to_delete $(echo ${group_array[*]} | awk '{$NF=""; print $0}')" done # Delete superfluous metadata: -echo "Deleting superfluous buffers & metadata ..." -segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($10 == "") print $1 }') ) -for s in ${segments[@]}; do rm $RESULTS_DIR/$s.*; done - +#segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($16 == "") print $1 }') ) +segments=( $(echo $to_delete) ) +echo "Deleting superfluous buffers & metadata (${segments[*]}) ..." +for s in ${segments[@]}; do rm $RESULTS_DIR/${s}.* ; done + +# Detecting Session-Id values: +grep ^sessionid= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/session-ids +if [ -s $RESULTS_DIR/session-ids ] +then + count=0 + while read -r line; do count=$((count+1)) ; echo "Detected Session-Id $count: $line"; done < $RESULTS_DIR/session-ids + rm $RESULTS_DIR/session-ids +fi + +# Detecting Origin-Host values: +grep ^originhost= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/origin-hosts +if [ -s $RESULTS_DIR/origin-hosts ] +then + count=0 + while read -r line; do count=$((count+1)) ; echo "Detected Origin-Host $count: $line"; done < $RESULTS_DIR/origin-hosts + #rm $RESULTS_DIR/origin-hosts +fi + +# Purge frames with Session-Id not wanted: +if [ -n "$SIDS_FILE" ] +then + grep -l -w -f $SIDS_FILE $RESULTS_DIR/*metadata > $RESULTS_DIR/.wanted + grep -l ^sessionid $RESULTS_DIR/*metadata > $RESULTS_DIR/.all + for file in `grep -vf $RESULTS_DIR/.wanted $RESULTS_DIR/.all` + do + frm=$(basename $file | cut -d\. -f1) + sid=$(grep ^sessionid= $file | cut -d= -f2-) + echo "Purge results for frame $frm (Session-Id: '$sid') ..." + rm $RESULTS_DIR/${frm}.* + done + rm $RESULTS_DIR/.wanted $RESULTS_DIR/.all +fi _exit "Done!" 0