3 # ANNA - Anna is Not Nothingness Anymore
5 # (c) Copyright 2005-2014 Eduardo Ramos Testillano & Francisco Ruiz Rayo
7 # http://redmine.teslayout.com/projects/anna-suite
9 # Redistribution and use in source and binary forms, with or without
10 # modification, are permitted provided that the following conditions
13 # * Redistributions of source code must retain the above copyright
14 # notice, this list of conditions and the following disclaimer.
15 # * Redistributions in binary form must reproduce the above
16 # copyright notice, this list of conditions and the following disclaimer
17 # in the documentation and/or other materials provided with the
19 # * Neither the name of the copyright holder nor the names of its
20 # contributors may be used to endorse or promote products derived from
21 # this software without specific prior written permission.
23 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 # Authors: eduardo.ramos.testillano@gmail.com
36 # cisco.tierra@gmail.com
39 # Decoder version using tshark tool
41 # RESTRICTIONS: only TCP datagrams. Allow split diameter messages between frames, but not, several messages into single datagram.
55 echo "Usage: $0 [-h|--help] [-o|--other-ports <list>] [-d|--results-dir <dir>] [-s|--sids <file>] <pcap>"
57 echo " -h|--help: this usage help."
58 echo " -o|--other-ports <list>: space-separated list of ports which frames"
59 echo " will be decoded as diameter protocol although"
60 echo " not being standard. For example, we could use"
61 echo " \"13868\" to disect the Ericsson Sy variant."
62 echo " -d|--results-dir <dir>: directory where results are stored."
63 echo " By default, pcap dirname."
64 echo " -s|--sids <file>: file containing a list of Session-Id values"
65 echo " (one per line) to be taken into account."
66 echo " If missing, all the frames will be extracted."
68 echo " pcap: pcap formatted file to be processed."
70 echo " The utility, dumps the extracted hexadecimal content and useful information"
71 echo " (timestamps, source, destination, etc.) within a metadata file:"
73 echo " <results directory>/<frame sequence>.hex"
74 echo " <results directory>/<frame sequence>.metadata"
85 while [ $# -gt 0 ]; do
93 [ -z "$OTHER_PORTS" ] && _exit "Missing non-standard ports list"
108 first=$(echo $1 | cut -c1)
109 [ "$first" = "-" ] && _exit "Unsupported script option: $1. Type '$SCR_BN -h' (or --help) to print the available options."
116 [ -z "$PCAP_FILE" ] && _exit "Missing pcap file"
117 [ ! -f "$PCAP_FILE" ] && _exit "Cannot found provided pcap file '$PCAP_FILE' !!"
118 [ -z "$RESULTS_DIR" ] && RESULTS_DIR=`dirname $PCAP_FILE`
119 [ ! -d $RESULTS_DIR ] && _exit "The results directory '$RESULTS_DIR' must exists !!"
120 if [ -n "$SIDS_FILE" ]
122 [ ! -f $SIDS_FILE ] && _exit "The Session-Id list file '$SIDS_FILE' provided, does not exist !!"
145 echo "============================================"
146 echo "Diameter buffer extractor from PCAP raw file"
147 echo "============================================"
151 [ "$1" = "" -o "$1" = "--help" -o "$1" = "-h" ] && usage
155 which $TSHARK >/dev/null 2>/dev/null
156 [ $? -ne 0 ] && _exit "Missing 'tshark' tool !!"
158 # Get the frames with diameter content (take care about '-2' two-pass option and don't add it, because we need to get reassembled parts in their corresponding frames):
159 # Fields needed (we won't need diameter.hopbyhopid & diameter.endtoendid to verify diameter message as hint patterns; length management will be enough): see https://www.wireshark.org/docs/dfref/d/diameter.html
160 FIELDS="-e frame.number -e frame.time_epoch -e ip.src_host -e ip.dst_host -e diameter.cmd.code -e diameter.flags.request -e diameter.applicationId -e diameter.hopbyhopid -e diameter.endtoendid -e diameter.Session-Id -e diameter.Origin-Host -e diameter.Subscription-Id-Data -e diameter.Subscription-Id-Type -e frame.len -e tcp.len -e diameter.length -e frame.protocols -e tcp.segment"
161 # Disect selectors for non-standard diameter ports:
162 for port in $OTHER_PORTS
164 echo "Taking tcp port $port to be decoded as diameter protocol"
165 DISECT_SELECTORS="$DISECT_SELECTORS -d tcp.port=$port,diameter"
168 $TSHARK -E separator="|" -r $PCAP_FILE -N mntC -Tfields $FIELDS $DISECT_SELECTORS 2>/dev/null | grep -i diameter > $tmpdir/diameter_frames
171 # frm timestamp src dst code R App-ID HopByHop EndToEnd Sid OHost Subs SubsType FRM TCP DIA protocol segments
172 # 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
173 # 1|tt.tt|gt_traf|vcbavipt|272|1|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|432|432|eth:ip:tcp:diameter:diameter:diameter3gpp|
174 # 3|tt.tt|vcbavipt|gt_traf|272|0|16777238|0x0004e6e6|0x000bd986|xxx|xxxxx|xxxxx|xxxxxxx|fff|292|292|eth:ip:tcp:diameter:diameter:diameter3gpp|
175 # 5|tt.tt|gt_traf|vcbavipt||||||ffff|1400||eth:ip:tcp:diameter|
176 # 6|tt.tt|gt_traf|vcbavipt|265|1|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|572|1972|eth:ip:tcp:diameter:diameter:diameter3gpp|5,6
177 # 8|tt.tt|vcbavipt|gt_traf|265|0|16777236|0x000c73c3|0x0004cee4|xxx|xxxxx|xxxxx|xxxxxxx|fff|248|248|eth:ip:tcp:diameter:diameter:diameter3gpp|
178 all_frames=( $(cat $tmpdir/diameter_frames | cut -d\| -f1) )
179 needs_join=( $(cat $tmpdir/diameter_frames | cut -d\| -f18) )
181 # Reassemble procedure (using frame 1 as example):
182 # (for non segmented frames, it is enough with tcp or diameter length within the frame content itself)
183 # 1) Get the TCP length: 432 bytes. 432*2 = 864 characters per byte in hexadecimal string format
184 # 2) Get the frame length: `wc -c $tmpdir/block.$frame` => 997
185 # 3) Get 864 from the tail: `cat $tmpdir/block.$frame | cut -c133
187 # Dump the hex blocks for all the diameter frames:
188 cat $PCAP_FILE | rawshark -s -r - -d proto:diameter -F data 2>/dev/null > $tmpdir/all_hex_data
189 for frame in ${all_frames[@]}; do
190 grep "^$frame " $tmpdir/all_hex_data | cut -d\" -f2 | sed 's/://g' > $tmpdir/block.$frame
191 frame_info=$(grep "^${frame}|" $tmpdir/diameter_frames)
193 ##########################################################################################
194 # Get the diameter part:
195 frm_len=$(echo $frame_info | cut -d\| -f14)
196 tcp_len=$(echo $frame_info | cut -d\| -f15)
197 dia_len=$(echo $frame_info | cut -d\| -f16)
198 protocol=$(echo $frame_info | cut -d\| -f17)
200 # TCP and SCTP supported:
201 transport=$(echo $protocol | grep -ow tcp)
202 [ -z "$transport" ] && transport=$(echo $protocol | grep -ow sctp)
206 cut_len=$((2*dia_len))
207 cat $tmpdir/block.$frame | rev | cut -c-${cut_len} | rev > $RESULTS_DIR/$frame.hex
211 cut_len=$((frm_len-tcp_len))
212 defi=$((2*cut_len + 1))
213 cat $tmpdir/block.$frame | cut -c${defi}- > $RESULTS_DIR/$frame.hex
214 cp $tmpdir/block.$frame /tmp/block
217 *) _exit "Only TCP and SCTP transports supported !!"
220 ##########################################################################################
222 echo -n "Created $RESULTS_DIR/$frame.hex"
225 ts=$(echo $frame_info | cut -d\| -f2)
227 src=$(echo $frame_info | cut -d\| -f3)
228 dst=$(echo $frame_info | cut -d\| -f4)
229 code=$(echo $frame_info | cut -d\| -f5)
230 isreq=$(echo $frame_info | cut -d\| -f6)
231 appid=$(echo $frame_info | cut -d\| -f7)
232 sid=$(echo $frame_info | cut -d\| -f10)
233 oh=$(echo $frame_info | cut -d\| -f11)
234 subscriber=$(echo $frame_info | cut -d\| -f12)
235 subscribertype=$(echo $frame_info | cut -d\| -f13)
236 [ "$subscribertype" = "0" ] && subscribertype=msisdn
237 [ "$subscribertype" = "1" ] && subscribertype=imsi
240 #hbh=$(echo $frame_info | cut -d\| -f8)
241 #e2e=$(echo $frame_info | cut -d\| -f9)
242 # HBH and ETE To decimal:
243 #hbh=$(printf "%d\n" $hbh)
244 #e2e=$(printf "%d\n" $e2e)
245 echo "date=$date" > $RESULTS_DIR/$frame.metadata
246 echo "timestamp=$ts" >> $RESULTS_DIR/$frame.metadata
247 echo "src=$src" >> $RESULTS_DIR/$frame.metadata
248 echo "dst=$dst" >> $RESULTS_DIR/$frame.metadata
249 echo "code=$code" >> $RESULTS_DIR/$frame.metadata
250 echo "isrequest=$isreq" >> $RESULTS_DIR/$frame.metadata
251 echo "applicationid=$appid" >> $RESULTS_DIR/$frame.metadata
252 [ -n "$sid" ] && echo "sessionid=$sid" >> $RESULTS_DIR/$frame.metadata
253 echo "originhost=$oh" >> $RESULTS_DIR/$frame.metadata
254 if [ -n "$subscriber" ]
256 echo "subscriber=$subscriber" >> $RESULTS_DIR/$frame.metadata
257 echo "subscribertype=$subscribertype" >> $RESULTS_DIR/$frame.metadata
259 #echo "hopbyhop=$hbh" >> $RESULTS_DIR/$frame.metadata
260 #echo "endtoend=$e2e" >> $RESULTS_DIR/$frame.metadata
262 echo " and $RESULTS_DIR/$frame.metadata"
265 # Join frames which need to be reassembled:
266 for group in ${needs_join[@]}; do
267 echo "Grouping frames $group ..."
268 group_array=( $(echo $group | sed 's/,/ /g') )
269 for frame in ${group_array[@]}; do
270 cat $RESULTS_DIR/$frame.hex >> $tmpdir/diam.$group
272 cat $tmpdir/diam.$group | tr -d '\n' > $RESULTS_DIR/$frame.hex
275 # Delete superfluous metadata:
276 echo "Deleting superfluous buffers & metadata ..."
277 segments=( $(cat $tmpdir/diameter_frames | awk -F\| '{ if ($16 == "") print $1 }') )
278 for s in ${segments[@]}; do rm $RESULTS_DIR/$s.*; done
280 # Detecting Session-Id values:
281 grep ^sessionid= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/session-ids
282 if [ -s $RESULTS_DIR/session-ids ]
285 while read -r line; do count=$((count+1)) ; echo "Detected Session-Id $count: $line"; done < $RESULTS_DIR/session-ids
286 rm $RESULTS_DIR/session-ids
289 # Detecting Origin-Host values:
290 grep ^originhost= $RESULTS_DIR/*.metadata 2>/dev/null | cut -d= -f2- | sort -u > $RESULTS_DIR/origin-hosts
291 if [ -s $RESULTS_DIR/origin-hosts ]
294 while read -r line; do count=$((count+1)) ; echo "Detected Origin-Host $count: $line"; done < $RESULTS_DIR/origin-hosts
295 #rm $RESULTS_DIR/origin-hosts
298 # Purge frames with Session-Id not wanted:
299 if [ -n "$SIDS_FILE" ]
301 grep -l -w -f $SIDS_FILE $RESULTS_DIR/*metadata > $RESULTS_DIR/.wanted
302 grep -l ^sessionid $RESULTS_DIR/*metadata > $RESULTS_DIR/.all
303 for file in `grep -vf $RESULTS_DIR/.wanted $RESULTS_DIR/.all`
305 frm=$(basename $file | cut -d\. -f1)
306 sid=$(grep ^sessionid= $file | cut -d= -f2-)
307 echo "Purge results for frame $frm (Session-Id: '$sid') ..."
308 rm $RESULTS_DIR/${frm}.*
310 rm $RESULTS_DIR/.wanted $RESULTS_DIR/.all