Delete superfluous metadata in SCTP

diff --git a/example/diameter/pcapDecoder/tsharkDecoder.sh b/example/diameter/pcapDecoder/tsharkDecoder.sh
index fbb6bb5..93b29d4 100755 (executable)
--- a/example/diameter/pcapDecoder/tsharkDecoder.sh
+++ b/example/diameter/pcapDecoder/tsharkDecoder.sh
@@ -157,7 +157,7 @@ which $TSHARK >/dev/null 2>/dev/null
 
 # Get the frames with diameter content (take care about '-2' two-pass option and don't add it, because we need to get reassembled parts in their corresponding frames):
 # Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): see https://www.wireshark.org/docs/dfref/d/diameter.html
-FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host -e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.Origin-Host -e diameter.Subscription-Id-Data -e diameter.Subscription-Id-Type -e frame.len -e tcp.len -e diameter.length -e frame.protocols -e tcp.segment"
+FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host -e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.Origin-Host -e diameter.Subscription-Id-Data -e diameter.Subscription-Id-Type -e frame.len -e tcp.len -e diameter.length -e frame.protocols -e tcp.segment -e sctp.chunk_length"
 # Disect selectors for non-standard diameter ports:
 for port in $OTHER_PORTS
 do
@@ -165,18 +165,38 @@ do
   DISECT_SELECTORS="$DISECT_SELECTORS -d tcp.port=$port,diameter"
 done
 
-$TSHARK -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames
+$TSHARK -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null > $tmpdir/all_frames
+grep -i diameter $tmpdir/all_frames > $tmpdir/diameter_frames
+#cat $tmpdir/all_frames
+
 # Example output:
 #                                                                                         / lengths \
-# frm timestamp src   dst   code R  App-ID   HopByHop   EndToEnd  Sid OHost Subs SubsType FRM TCP DIA   protocol                                segments
-#   1   2      3       4      5  6    7          8          9     10   11    12     13    14   15  16      17                                      18
+# frm timestamp src   dst   code R  App-ID   HopByHop   EndToEnd  Sid OHost Subs SubsType FRM TCP DIA   protocol                              segments    sctp chunk length
+#   1   2      3       4      5  6    7          8          9     10   11    12     13    14   15  16      17                                    18             19
 #   1|tt.tt|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp|
 #   3|tt.tt|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp|
 #   5|tt.tt|gt_traf|vcbavipt||||||ffff|1400||eth:ip:tcp:diameter|
 #   6|tt.tt|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|572|1972|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6
 #   8|tt.tt|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp|
-all_frames=( $(cat $tmpdir/diameter_frames | cut -d\| -f1) )
-needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f18) )
+cat $tmpdir/diameter_frames | cut -d\| -f1 > $tmpdir/frames_num
+all_frames=( $(cat $tmpdir/frames_num) )
+
+# Segments groups (TCP or SCTP):
+segmentsGroups=
+
+# The case of SCTP is precalculated:
+tshark -nr $PCAP_FILE $DISECT_SELECTORS -Tfields -e frame.number -e sctp.chunk_flags 2>/dev/null | awk '{ if ( $2 !~ "0x03" && $2 != "" ) print $1 }' > $tmpdir/extra_frames_num
+if [ -s $tmpdir/extra_frames_num ]
+then
+  # Take into account these frames in all_frames array:
+
+  cat $tmpdir/extra_frames_num >> $tmpdir/frames_num
+  all_frames=( $(cat $tmpdir/frames_num | sort -un) )
+
+  # Prepare as groups (1,2 7,8 ...)
+  segmentsGroups=$(tshark -nr $PCAP_FILE $DISECT_SELECTORS -Tfields -e frame.number -e sctp.chunk_flags 2>/dev/null | awk '{ if ( $2 !~ "0x03" && $2 != "" ) print $1; else print "\n" }' | uniq | tr '\n' ',' | sed 's/,,/ /g')
+fi
+
 
 # Reassemble procedure (using frame 1 as example):
 # (for non segmented frames, it is enough with tcp or diameter length within the frame content itself)
@@ -200,18 +220,36 @@ for frame in ${all_frames[@]}; do
   # TCP and SCTP supported:
   transport=$(echo $protocol | grep -ow tcp)
   [ -z "$transport" ] && transport=$(echo $protocol | grep -ow sctp)
+  # in case of segmented SCTP frames, protocol won't be sctp ...:
+  if [ -s "$tmpdir/extra_frames_num" -a -z "$transport" ]
+  then
+    grep -qw ^${frame} $tmpdir/extra_frames_num
+    if [ $? -eq 0 ]
+    then
+      transport=sctp
+      frame_info=$(grep "^${frame}|" $tmpdir/all_frames)
+    fi
+  fi
 
   case $transport in
     sctp)
+      chunk_length=$(echo $frame_info | cut -d\| -f19)
+      dia_partial_len=$((chunk_length-16))
+      if [ "$dia_len" != "$dia_partial_len" ]
+      then
+        #echo "Segmented SCTP frame $frame; taking partial diameter length = $dia_partial_len"
+        dia_len=$dia_partial_len
+      fi 
       cut_len=$((2*dia_len))
       cat $tmpdir/block.$frame | rev | cut -c-${cut_len} | rev > $RESULTS_DIR/$frame.hex
       ;;
 
     tcp)
-      cut_len=$((frm_len-tcp_len))
-      defi=$((2*cut_len + 1))
-      cat $tmpdir/block.$frame | cut -c${defi}- > $RESULTS_DIR/$frame.hex
+      cut_len=$((2*(frm_len-tcp_len) + 1))
+      cat $tmpdir/block.$frame | cut -c${cut_len}- > $RESULTS_DIR/$frame.hex
       cp $tmpdir/block.$frame /tmp/block
+      segments=$(echo $frame_info | cut -d\| -f18)
+      segmentsGroups="$segmentsGroups $segments"
       ;;
 
     *) _exit "Only TCP and SCTP transports supported !!"
@@ -263,19 +301,24 @@ for frame in ${all_frames[@]}; do
 done
 
 # Join frames which need to be reassembled:
-for group in ${needs_join[@]}; do
-  echo "Grouping frames $group ..."
+to_delete=
+for group in $segmentsGroups
+do
   group_array=( $(echo $group | sed 's/,/ /g') )
+  echo "Grouping frames ${group_array[*]} ..."
   for frame in ${group_array[@]}; do
     cat $RESULTS_DIR/$frame.hex >> $tmpdir/diam.$group
   done
   cat $tmpdir/diam.$group | tr -d '\n' > $RESULTS_DIR/$frame.hex
+  # Delete all frames except last one in the group:
+  to_delete="$to_delete  $(echo ${group_array[*]} | awk '{$NF=""; print $0}')"
 done
 
 # Delete superfluous metadata:
-echo "Deleting superfluous buffers & metadata ..."
-segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($16 == "") print $1 }') )
-for s in ${segments[@]}; do rm $RESULTS_DIR/$s.*; done
+#segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($16 == "") print $1 }') )
+segments=( $(echo $to_delete) )
+echo "Deleting superfluous buffers & metadata (${segments[*]}) ..."
+for s in ${segments[@]}; do rm $RESULTS_DIR/${s}.* ; done
 
 # Detecting Session-Id values:
 grep ^sessionid= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/session-ids