# Decoder version using tshark tool
+#
+# RESTRICTIONS: TCP & SCTP datagrams. Allow split diameter messages between frames, but not, several messages into single datagram.
+# If sniffing fails, try to create the <frame>.hex and <frame>.metadata files manually (try 'tshark -nr input.pcap -Vx' and so on ...)
#############
# VARIABLES #
#############
tmpdir=$(mktemp -d)
+TSHARK=tshark
#############
# FUNCTIONS #
echo
echo " pcap: pcap formatted file to be processed."
echo
- echo " The utility, dumps the extracted hexadecimal content"
- echo " and useful information as timestamps, source and"
- echo " destination:"
+ echo " The utility, dumps the extracted hexadecimal content and useful information"
+ echo " (timestamps, source, destination, etc.) within a metadata file:"
+ echo
echo " <results directory>/<frame sequence>.hex"
echo " <results directory>/<frame sequence>.metadata"
echo
parse_arguments "$@"
# Tshark available:
-which tshark >/dev/null
+which $TSHARK >/dev/null 2>/dev/null
[ $? -ne 0 ] && _exit "Missing 'tshark' tool !!"
# Get the frames with diameter content (take care about '-2' two-pass option and don't add it, because we need to get reassembled parts in their corresponding frames):
# Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): see https://www.wireshark.org/docs/dfref/d/diameter.html
-FIELDS_DIAMETER="-e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.length"
-FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host $FIELDS_DIAMETER -e tcp.len -e frame.protocols -e tcp.segment"
+FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host -e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.Origin-Host -e diameter.Subscription-Id-Data -e diameter.Subscription-Id-Type -e frame.len -e tcp.len -e diameter.length -e frame.protocols -e tcp.segment -e sctp.chunk_length"
# Disect selectors for non-standard diameter ports:
for port in $OTHER_PORTS
do
DISECT_SELECTORS="$DISECT_SELECTORS -d tcp.port=$port,diameter"
done
-tshark -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames
+$TSHARK -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null > $tmpdir/all_frames
+grep -i diameter $tmpdir/all_frames > $tmpdir/diameter_frames
+#cat $tmpdir/all_frames
+
# Example output:
-# /length\
-# frame timestamp src dst code R App-ID HopByHop EndToEnd Session-Id DIAM TCP protocol segments
-# 1 2 3 4 5 6 7 8 9 10 11 12 13 14
-# 1|1427215933.697904000|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxxxxxx|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp|
-# 3|1427215934.449523000|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxxxxxx|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp|
-# 5|1427215934.456160000|gt_traf|vcbavipt||||||||1400|eth:ip:tcp:diameter|
-# 6|1427215934.456204000|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxxxxxx|1972|572|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6
-# 8|1427215935.123559000|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxxxxxx|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp|
-all_frames=( $(cat $tmpdir/diameter_frames | cut -d\| -f1) )
-needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f14) )
-main_frames=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($12 != "") print $1 }') )
+# / lengths \
+# frm timestamp src dst code R App-ID HopByHop EndToEnd Sid OHost Subs SubsType FRM TCP DIA protocol segments sctp chunk length
+# 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
+# 1|tt.tt|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp|
+# 3|tt.tt|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp|
+# 5|tt.tt|gt_traf|vcbavipt||||||ffff|1400||eth:ip:tcp:diameter|
+# 6|tt.tt|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|572|1972|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6
+# 8|tt.tt|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp|
+cat $tmpdir/diameter_frames | cut -d\| -f1 > $tmpdir/frames_num
+all_frames=( $(cat $tmpdir/frames_num) )
+
+# Segments groups (TCP or SCTP):
+segmentsGroups=
+
+# The case of SCTP is precalculated:
+tshark -nr $PCAP_FILE $DISECT_SELECTORS -Tfields -e frame.number -e sctp.chunk_flags 2>/dev/null | awk '{ if ( $2 !~ "0x03" && $2 != "" ) print $1 }' > $tmpdir/extra_frames_num
+if [ -s $tmpdir/extra_frames_num ]
+then
+ # Take into account these frames in all_frames array:
+
+ cat $tmpdir/extra_frames_num >> $tmpdir/frames_num
+ all_frames=( $(cat $tmpdir/frames_num | sort -un) )
+
+ # Prepare as groups (1,2 7,8 ...)
+ segmentsGroups=$(tshark -nr $PCAP_FILE $DISECT_SELECTORS -Tfields -e frame.number -e sctp.chunk_flags 2>/dev/null | awk '{ if ( $2 !~ "0x03" && $2 != "" ) print $1; else print "\n" }' | uniq | tr '\n' ',' | sed 's/,,/ /g')
+fi
+
# Reassemble procedure (using frame 1 as example):
# (for non segmented frames, it is enough with tcp or diameter length within the frame content itself)
grep "^$frame " $tmpdir/all_hex_data | cut -d\" -f2 | sed 's/://g' > $tmpdir/block.$frame
frame_info=$(grep "^${frame}|" $tmpdir/diameter_frames)
+ ##########################################################################################
# Get the diameter part:
- tcp_len=$(echo $frame_info | cut -d\| -f12)
- frm_len=$(wc -c $tmpdir/block.$frame | awk '{ print $1 }')
- cut_len=$((frm_len-2*tcp_len))
- cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex
+ frm_len=$(echo $frame_info | cut -d\| -f14)
+ tcp_len=$(echo $frame_info | cut -d\| -f15)
+ dia_len=$(echo $frame_info | cut -d\| -f16)
+ protocol=$(echo $frame_info | cut -d\| -f17)
+
+ # TCP and SCTP supported:
+ transport=$(echo $protocol | grep -ow tcp)
+ [ -z "$transport" ] && transport=$(echo $protocol | grep -ow sctp)
+ # in case of segmented SCTP frames, protocol won't be sctp ...:
+ if [ -s "$tmpdir/extra_frames_num" -a -z "$transport" ]
+ then
+ grep -qw ^${frame} $tmpdir/extra_frames_num
+ if [ $? -eq 0 ]
+ then
+ transport=sctp
+ frame_info=$(grep "^${frame}|" $tmpdir/all_frames)
+ fi
+ fi
+
+ case $transport in
+ sctp)
+ chunk_length=$(echo $frame_info | cut -d\| -f19)
+ dia_partial_len=$((chunk_length-16))
+ if [ "$dia_len" != "$dia_partial_len" ]
+ then
+ #echo "Segmented SCTP frame $frame; taking partial diameter length = $dia_partial_len"
+ dia_len=$dia_partial_len
+ fi
+ cut_len=$((2*dia_len))
+ cat $tmpdir/block.$frame | rev | cut -c-${cut_len} | rev > $RESULTS_DIR/$frame.hex
+ ;;
+
+ tcp)
+ cut_len=$((2*(frm_len-tcp_len) + 1))
+ cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex
+ cp $tmpdir/block.$frame /tmp/block
+ segments=$(echo $frame_info | cut -d\| -f18)
+ segmentsGroups="$segmentsGroups $segments"
+ ;;
+
+ *) _exit "Only TCP and SCTP transports supported !!"
+ ;;
+ esac
+ ##########################################################################################
+
echo -n "Created $RESULTS_DIR/$frame.hex"
# Metadata:
isreq=$(echo $frame_info | cut -d\| -f6)
appid=$(echo $frame_info | cut -d\| -f7)
sid=$(echo $frame_info | cut -d\| -f10)
+ oh=$(echo $frame_info | cut -d\| -f11)
+ subscriber=$(echo $frame_info | cut -d\| -f12)
+ subscribertype=$(echo $frame_info | cut -d\| -f13)
+ [ "$subscribertype" = "0" ] && subscribertype=msisdn
+ [ "$subscribertype" = "1" ] && subscribertype=imsi
+
+
#hbh=$(echo $frame_info | cut -d\| -f8)
#e2e=$(echo $frame_info | cut -d\| -f9)
# HBH and ETE To decimal:
echo "isrequest=$isreq" >> $RESULTS_DIR/$frame.metadata
echo "applicationid=$appid" >> $RESULTS_DIR/$frame.metadata
[ -n "$sid" ] && echo "sessionid=$sid" >> $RESULTS_DIR/$frame.metadata
+ echo "originhost=$oh" >> $RESULTS_DIR/$frame.metadata
+ if [ -n "$subscriber" ]
+ then
+ echo "subscriber=$subscriber" >> $RESULTS_DIR/$frame.metadata
+ echo "subscribertype=$subscribertype" >> $RESULTS_DIR/$frame.metadata
+ fi
#echo "hopbyhop=$hbh" >> $RESULTS_DIR/$frame.metadata
#echo "endtoend=$e2e" >> $RESULTS_DIR/$frame.metadata
done
# Join frames which need to be reassembled:
-for group in ${needs_join[@]}; do
- echo "Grouping frames $group ..."
+to_delete=
+for group in $segmentsGroups
+do
group_array=( $(echo $group | sed 's/,/ /g') )
+ echo "Grouping frames ${group_array[*]} ..."
for frame in ${group_array[@]}; do
- cat $RESULTS_DIR/$frame.hex >> $tmpdir/diam.$group
+ cat $RESULTS_DIR/$frame.hex >> $tmpdir/diam.$group 2>/dev/null
done
cat $tmpdir/diam.$group | tr -d '\n' > $RESULTS_DIR/$frame.hex
+ # Delete all frames except last one in the group:
+ to_delete="$to_delete $(echo ${group_array[*]} | awk '{$NF=""; print $0}')"
done
# Delete superfluous metadata:
-echo "Deleting superfluous buffers & metadata ..."
-segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($11 == "") print $1 }') )
-for s in ${segments[@]}; do rm $RESULTS_DIR/$s.*; done
+#segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($16 == "") print $1 }') )
+segments=( $(echo $to_delete) )
+echo "Deleting superfluous buffers & metadata (${segments[*]}) ..."
+for s in ${segments[@]}; do rm -f $RESULTS_DIR/${s}.* ; done
# Detecting Session-Id values:
grep ^sessionid= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/session-ids
if [ -s $RESULTS_DIR/session-ids ]
then
count=0
- while read -r line; do count=$((count+1)) ; echo "Detected Session-Id $count: $line"; done < $RESULTS_DIR/session-ids
+ while read -r line; do count=$((count+1)) ; echo "Detected Session-Id $count: $line"; done < $RESULTS_DIR/session-ids
rm $RESULTS_DIR/session-ids
fi
+# Detecting Origin-Host values:
+grep ^originhost= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/origin-hosts
+if [ -s $RESULTS_DIR/origin-hosts ]
+then
+ count=0
+ while read -r line; do count=$((count+1)) ; echo "Detected Origin-Host $count: $line"; done < $RESULTS_DIR/origin-hosts
+ #rm $RESULTS_DIR/origin-hosts
+fi
+
# Purge frames with Session-Id not wanted:
if [ -n "$SIDS_FILE" ]
then
rm $RESULTS_DIR/.wanted $RESULTS_DIR/.all
fi
+# Purge frames with invalid metadata:
+invalid=( $(grep -lw "^code=$" $RESULTS_DIR/*metadata) )
+for file in ${invalid[@]}
+do
+ frm=$(basename $file | cut -d\. -f1)
+ rm $RESULTS_DIR/${frm}.*
+done
+
_exit "Done!" 0