#!/bin/bash
#
# simple script to monitor netstat output
#

FIRST_RUN=0

# semaphor file
#
SEMAPHOR_FILE="/tmp/stop_monitor_netstat.sh"

# log files
#
LOGDIR="/var/log/monitor_netstat"

# general script log file
#
LOGFILE="${LOGDIR}/monitor_network_statistics.log"

# logfile for the monitors
#
IOSTAT_LOGFILE="${LOGDIR}/monitor_iostat.log"
IFSTAT_LOGFILE="${LOGDIR}/monitor_ifstat.log"
MPSTAT_LOGFILE="${LOGDIR}/monitor_mpstat.log"
VMSTAT_LOGFILE="${LOGDIR}/monitor_vmstat.log"    
NETSTAT_LOGFILE="${LOGDIR}/monitor_netstat.log"    
TOP_LOGFILE="${LOGDIR}/monitor_top.log"

# monitor intervall and count
#
MONITOR_INTERVALL=5
MONITOR_COUNT=720

(( LOOP_TIME = ${MONITOR_COUNT} * ${MONITOR_INTERVALL} + 60 ))

# monitor process pids / session ids
#
PIDS=""
SESSION_IDS=""

# netstat error counter and netstat package counter
#
CUR_packets_pruned=0
CUR_TCP_data_loss_events=0
CUR_packets_collapsed=0
CUR_timestamp=""
CUR_segments_received=0

PREV_packets_pruned=0
PREV_TCP_data_loss_events=0
PREV_packets_collapsed=0
PREV_timestamp=""
PREV_segments_received=0

# binaries and scripts
#
ADDTIMESTAMP="/root/dist/scripts/addtimestamp"

IOSTAT="/usr/bin/iostat"
IFSTAT="/root/dist/bin/64/ifstat"
MPSTAT="/usr/bin/mpstat"
VMSTAT="/usr/bin/vmstat"
TOP="/usr/bin/top"
NETSTAT="/bin/netstat"
BASH="/bin/bash"

# general variables

#
# for debugging only!
#
# PREFIX="echo "
 PREFIX=""

# script related variables
#
THISSCRIPT="$( basename $0 )"

# general functions
#
logMsg () {
  typeset CURMSG="[$(date +"%d.%m.%Y %H:%M:%S") ${THISSCRIPT}]  $* "
  [ "${LOGFILE}"x != ""x ] && echo "${CURMSG}" >>"${LOGFILE}"
  echo "${CURMSG}"
}

logErrorMsg () {
  logMsg "ERROR: $*"
}

logWarningMsg () {
  logMsg "WARNING: $*"
}

die () {
  typeset THISRC=$1

  if [ $# -ne 0 ] ; then
    shift
    [ ${THISRC} = 0 ] && logMsg $* || logErrorMsg $*
  fi
  exit ${THISRC}
}

# script special functions
#
start_monitor() {
    typeset CUR_PID=
    typeset CUR_SESS_ID=""
    
    ${BASH} -c "${COMMAND} | /usr/bin/awk '{\"date \\\"+%d.%m.%Y %H:%M:%S\\\" #\"|getline date; close(\"date \\\"+%d.%m.%Y %H:%M:%S\\\" #\"); printf (\"%s %s\n\", date, \$0)}'" 2>>"${LOGFILE}" >"${CUR_LOGFILE}"  &
    if [ $? -eq 0 ] ; then  
      CUR_PID=$!
      CUR_SESS_ID="$( ps -h -p ${CUR_PID} -o sess 2>/dev/null )" 
      echo " ${SESSION_IDS} " | grep " ${CUR_SESS_ID} " >/dev/null     
      [ $? -ne 0 ] && SESSION_IDS="${SESSION_IDS} ${CUR_SESS_ID} "
    fi
    echo "Session IDs: ${SESSION_IDS}"    
 }

kill_running_monitors() {
  typeset i
  typeset PIDS=""

# stop the monitors if they are still running
  if [ "${SESSION_IDS}"x != ""x ] ; then
    logMsg "Stopping the monitor processes(Session IDs: ${SESSION_IDS}) ..."
    for i in ${SESSION_IDS} ; do
      PIDS="$( ps  -s $i -o pid| grep -v PID )"

      for k in ${PIDS} ; do
        ps -p $k 2>/dev/null 1>/dev/null || continue
        if [ $k -ne $$ ] ; then
          logMsg "  Killing the process \"${k}\" ..."
          kill $k 2>/dev/null
        fi
      done
    done
    SESSION_IDS=""
  fi
}

# house keeping
#
cleanup() {
  logMsg "External signal detected ... stopping the logging ..."
  kill_running_monitors
  exit 0
}

# create the log directory if neccessary
#
mkdir -p "${LOGDIR}" || die 5 "Can not create the log directory \"${LOGDIR}\" "
cd "${LOGDIR}" || die 10 "Can not change the working directory to directory \"${LOGDIR}\" "

# check write access in the log direcctory and backup an old logfile if it exists
#
if [ -r "${LOGFILE}" ] ; then
  OLD_LOGFILE_NAME="${LOGFILE}.$( head -1 "${LOGFILE}" | cut -c2-20 )-$( tail -1 "${LOGFILE}" | cut -c2-20 )"  
  mv "${LOGFILE}" "${OLD_LOGFILE_NAME}" || die 15 "Can not rename the existing logfile \"${LOGFILE}\" "
else
  OLD_LOGFILE_NAME=""  
fi

touch "${LOGFILE}" || die 20 "Can not write to the logfile \"${LOGFILE}\" "
  
logMsg "${THISSCRIPT} started ..."
[ "${OLD_LOGFILE_NAME}"x != ""x ] && logMsg "Old log file renamed to \"${OLD_LOGFILE_NAME}\" "

# install trap handler
#
trap "cleanup" 1
trap "cleanup" 2
trap "cleanup" 3
trap "cleanup" 15

# remove an existing stop file
#
if [ -r "${SEMAPHOR_FILE}" ] ; then
  logMsg "Removing the stop semaphor file \"${SEMAPHOR_FILE}\" ..."
  rm -f "${SEMAPHOR_FILE}"
  logMsg "... ${THISSCRIPT} ended."
fi  


# main loop 
#
while [ 0 = 0 ] ; do

  kill_running_monitors

  # check for stop semaphor
  [ -r "${SEMAPHOR_FILE}" ] && die 0 " \"${SEMAPHOR_FILE}\" found - stopping execution"

# get netstat values
  CUR_timestamp="$( date )"
  logMsg "Retrieving current netstat values at ${CUR_timestamp} ..."
  ${NETSTAT} -s > "${NETSTAT_LOGFILE}"
  if [ $? -ne 0 ] ; then
    logErrorMsg "  Error retrieving netstat values -- skipping this round"
  else
    CUR_packets_pruned=$( egrep "packets pruned from receive queue because of socket buffer overrun"  "${NETSTAT_LOGFILE}"| awk '{ print $1 };' )
    CUR_TCP_data_loss_events=$( egrep "TCP data loss events"  "${NETSTAT_LOGFILE}"| awk '{ print $1 };' )
    CUR_packets_collapsed=$( egrep "packets collapsed in receive queue due to low socket buffer"  "${NETSTAT_LOGFILE}"| awk '{ print $1 };' )
    CUR_segments_received=$( egrep "[0-9] segments received"  "${NETSTAT_LOGFILE}"| awk '{ print $1 };' )
    logMsg "  segments received: ${CUR_segments_received}, packets pruned: ${CUR_packets_pruned} , TCP data loss events: ${CUR_TCP_data_loss_events}, packets collapsed: ${CUR_packets_collapsed}"
    
    if [ ${FIRST_RUN} = 0 ] ; then
      FIRST_RUN=1
    elif [ "${CUR_packets_pruned}" != "${PREV_packets_pruned}" -o \
           "${CUR_TCP_data_loss_events}" !=  "${PREV_TCP_data_loss_events}" -o \
           "${CUR_packets_collapsed}" !=  "${PREV_packets_collapsed}" ] ; then
      LOGFILE_PREFIX="$( date +%d.%m.%Y_%H:%M:%S )"
      logMsg "  One or more error counter changed -- saving the log files (file prefix: ${LOGFILE_PREFIX} )..."

      [ -r "${IOSTAT_LOGFILE}" ] && mv  "${IOSTAT_LOGFILE}" "${IOSTAT_LOGFILE}.${LOGFILE_PREFIX}"
      [ -r "${IFSTAT_LOGFILE}" ] && mv  "${IFSTAT_LOGFILE}" "${IFSTAT_LOGFILE}.${LOGFILE_PREFIX}"
      [ -r "${MPSTAT_LOGFILE}" ] && mv  "${MPSTAT_LOGFILE}" "${MPSTAT_LOGFILE}.${LOGFILE_PREFIX}"
      [ -r "${VMSTAT_LOGFILE}" ] && mv  "${VMSTAT_LOGFILE}" "${VMSTAT_LOGFILE}.${LOGFILE_PREFIX}"
      [ -r "${TOP_LOGFILE}" ] && mv  "${TOP_LOGFILE}" "${TOPSTAT_LOGFILE}.${LOGFILE_PREFIX}"
      cat "${NETSTAT_LOGFILE}" | ${ADDTIMESTAMP} >"${NETSTAT_LOGFILE}.${LOGFILE_PREFIX}"
    else
      logMsg "  Error counter did not change - removing the log files ..."
      rm -f "${IOSTAT_LOGFILE}" "${IFSTAT_LOGFILE}" "${MPSTAT_LOGFILE}" "${VMSTAT_LOGFILE}" "${TOP_LOGFILE}" "${NETSTAT_LOGFILE}"
    fi

    PREV_packets_pruned=${CUR_packets_pruned}
    PREV_TCP_data_loss_events=${CUR_TCP_data_loss_events}
    PREV_packets_collapsed=${CUR_packets_collapsed}
    PREV_timestamp=${CUR_timestamp}
    PREV_segments_received=${CUR_segments_received}

    logMsg "  Restarting the monitors ..."
    
    logMsg "    Starting iostat ..."
    COMMAND="${IOSTAT} ${MONITOR_INTERVALL} ${MONITOR_COUNT} -xt"
    CUR_LOGFILE="${IOSTAT_LOGFILE}"
    start_monitor

    logMsg "    Starting ifstat ..."   
    COMMAND="${IFSTAT} -a ${MONITOR_INTERVALL} ${MONITOR_COUNT}" 
    CUR_LOGFILE="${IFSTAT_LOGFILE}"
    start_monitor 
    
    logMsg "    Starting mpstat ..."    
    CUR_LOGFILE="${MPSTAT_LOGFILE}"
    COMMAND="${MPSTAT} -P ALL ${MONITOR_INTERVALL} ${MONITOR_COUNT}"
    start_monitor

    logMsg "    Starting vmstat ..."    
    CUR_LOGFILE="${VMSTAT_LOGFILE}"    
    COMMAND="${VMSTAT} ${MONITOR_INTERVALL} ${MONITOR_COUNT}"
    start_monitor

    logMsg "    Starting top ..."
    CUR_LOGFILE="${TOP_LOGFILE}"
    COMMAND="${TOP} -b -d ${MONITOR_INTERVALL} -n ${MONITOR_COUNT}"
    start_monitor
    
    logMsg "  Session IDs of the background processes are: ${SESSION_IDS}"
  fi
  logMsg "  Waiting ${LOOP_TIME} seconds before doing the next check ...."
  (( MAIN_LOOP_COUNT = ${LOOP_TIME} / 10 ))
  k=0
  while [ ${k} -lt ${MAIN_LOOP_COUNT} ] ; do
    if [ -r "${SEMAPHOR_FILE}" ] ; then     
      kill_running_monitors
      die 0 " \"${SEMAPHOR_FILE}\" found - stopping execution"
    fi
    (( k = k + 1 ))
    sleep 10
  done
  
done