#!/bin/bash
##系统工具箱脚本
##目前包括的功能有:
##系统负载;cpu使用率;内存使用率;swap使用情况;处于D/R状态的进程;系统inode号占用情况
##系统目录占用大小;系统io使用情况

##1.对输入的参数进行判断
##如果输入的参数为空或是help的话，会输出说明
if [ "$1" = "" ] || [ "$1" = "help" ];then
  cat <<EOF
Usage: systatus [ options ] num1 num2
Options:
    all: print all parameters status
    clo: print "cpu load" status
    cuse: print "cpu used" status
    muse: print "memory used" status
    swu: print "swap used" status
    psdr: print "process status in D&&R" status
    ino: print "system inode used" status
    size: print "directory size status(/,secondary directory,top3)" status
    iot: print "io usage(disk,process)" status 
    fde: print "system file descriptor" status
num1: systools execution interval of time
num2: systools number of executions
EOF
  exit 10
fi

sysver=1.1

##2.判断参数是否符合要求,不符合要求进行提示并退出执行
##1).arrA:输入的所有参数
##2).脚本规定的所有参数
arrA=($1 $2 $3 $4 $5 $6 $7 $8 $9 ${10} ${11} ${12})
arrB=(clo cuse muse swu psdr ino size iot fde all)

##3).判断$1是否为数字,是数字的话退出脚本
junum0=`echo $1 | sed s/[0-9]//g`
if [ ! -n "$junum0" ];then
  echo "please print value"
  exit 10
fi

##4).判断输入的参数是否为脚本规定的参数,不是的话退出脚本执行
for i in ${arrA[*]};do
  juvalue=`echo "${arrB[*]}" | grep -w "$i"`
  junum1=`echo $i | sed 's/[0-9]//g'`
  if [ $i = "ver" ];then
    echo "systool version: $sysver"
    exit 10
  fi
  if [ -n "$juvalue" ] || [ ! -n "$junum1" ];then
    :
  else
    echo "$i" is not exist
    exit 10
  fi
done

##5).判断指令格式是否正确,不正确退出脚本执行
##抓取执行间隔时间,执行次数的信息
vanum=0
fornum=1
for j in ${arrA[*]};do
  juva=`echo "$j" | sed 's/[0-9]//g'` 
  if [ -n "$juva" ] && [ "$vanum" = 1 ];then
    echo "print value error"
    exit 10
  elif [ -n "$j" ] && [ "$vanum" = 2 ];then
    echo "print value error"
    exit 10
  fi
  
  if [ "$j" = "all" ] && [ "$fornum" = 2 ];then
    echo "print value error: \"all\" is not first"
    exit 10
  fi

  if [[ $j =~ [0-9] ]];then
    let vanum++
    if [ "$vanum" = 1 ];then
      extime=$j
    elif [ "$vanum" = 2 ];then
      lofre=$j
    fi
  fi
let fornum++
done
if [ "$vanum" = 0 ];then
  extime=5
  lofre=5
elif [ "$vanum" = 1 ];then
  lofre=5
fi

##3.本次脚本执行生成的日志
##1).系统状态输出的日志
logname="/var/log/systatus/systatus.log-$(date +%F-%H-%M-%S)"
##2).工具箱运行状态日志
toolslog="/var/sysadmin_toolbox/other/systools/logs/systool.log-`date +%F`"

##3).工具箱运行日志
if [ ! -d "/var/sysadmin_toolbox/other/systools/logs" ];then
  mkdir -p /var/sysadmin_toolbox/other/systools/logs
fi

##4).建立日志文件目录
if [ ! -d "/var/log/systatus" ];then
  mkdir -p /var/log/systatus
fi

##4.建立相关目录
##建立临时目录,供存放相关数据进行分析用
if [ ! -d "/var/sysadmin_toolbox/other/systools/tmp" ];then
  mkdir -p /var/sysadmin_toolbox/other/systools/tmp
fi

##脚本执行pid号
pid_lo=`echo $$`

##本次执行生成的临时文件存放目录名称
tempdir="/var/sysadmin_toolbox/other/systools/tmp/systmp_`date +%F-%H-%M-%S`"
mkdir $tempdir

##5.识别信号
#trap "rm -rf $tempdir && exit 10" 2
##1).HUP信号,远程连接关闭继续在后台运行
trap ":" 1

#trap "rm -rf $tempdir" EXIT TSTP TERM

##2).TERM TSTP EXIT信号,删除临时文件;输出脚本退出的信息（命令行,日志）
trap "rm -rf $tempdir &&
      echo "" &&
      echo 'The script running is exit,please check log in details: ' && 
      echo $toolslog &&
      echo 'Warn:Systools exit running''(pid: $pid_lo)' >> $toolslog &&
      if [ -f "$logname" ];then
        echo 'systatus log file:' `echo $logname`
        echo 'systatus log location: $logname (pid: $pid_lo)' >> $toolslog   
      fi &&
      exit 10" TERM TSTP EXIT

echo -e "\033[1;32m##############system status check begin###############\033[0m"
echo ""
echo "Systools is running begin,time: `date +%F_%H:%M:%S` (pid: $pid_lo)" >> $toolslog
sleep 2

##5.判断根分区剩余容量是否符合要求
##1).剩余空间50M以下退出脚本运行
echo -e "\033[1msystools space check: \033[0m"
echo "systools space check:" >> $toolslog
space=`df -m / | awk '{print $4}' | tail -1`
if [ "$space" -le 50 ];then 
  echo "Root partition free space: !!abnormal!!"
  echo "Error: Root partition free space: abnormal (pid: $pid_lo)" >> $toolslog
  exit 10
else 
  echo "Root partition free space: ok"
  echo "Root partition free space: ok (pid: $pid_lo)" >> $toolslog
fi

##2).判断目录大小,大于50M停止运行
if [ -d "/var/log/systatus/" ];then
  dirsize=`du -s /var/log/systatus | awk '{print $1}'` 
  if [ "$dirsize" -ge 51200 ];then
    echo "systatus log director size: !!abnormal!!"
    echo "Error: Systatus log director size: abnormal (pid: $pid_lo)" >> $toolslog
    exit 10
  else
    echo "systatus log director size: ok"
    echo "systatus log director size: ok (pid: $pid_lo)" >> $toolslog
  fi
fi
echo ""
sleep 2

##6.判断是否有之前执行的脚本进程未退出
##对脚本运行的进程号进行过滤
##由执行用户决定是否杀掉之前长期未结束的相关进程
##使用kill -USR1将之前执行的脚本进程停止
oldtmp=`ls /var/sysadmin_toolbox/other/systools/tmp/ | wc -l`
protmp="$tempdir/pro.tmp"
ps -ef | grep 'systatus' | grep -v 'grep' | grep -v "$pid_lo" >> $protmp
pid_new=`cat $protmp | awk '{print $2}'`
if [ -n "$pid_new" ];then
  echo "old systatus process:"
  cat $protmp | sed '/^$/d'
  read -p "process is exist,whether to kill[y/n]" jucon
  for i in $pid_new;do
    if [ "$jucon" = y ];then
      kill -USR1 $i
      sleep 5
      ps -ef | grep 'systatus' | grep -v 'grep' | grep -w "$i" &>/dev/null
      if [ $? -ne 0 ];then
        echo "pid $i: is stopped in single \"USR1\""  ##single USR1
        echo "pid $i is stoped (single:USR1)" >> $toolslog
        sleep 2
      else
        kill -9 $i
        if [ $? -eq 0 ];then
          echo "$i is killed in single 9"  ##single9,kill,kill -USR2不起作用时执行
          echo "pid:$i is killed (single:9)" >> $toolslog
          sleep 2
        else
          echo "!!$i: is not killed,please check!!"
          echo "Error: pid $i not to killed" >> $toolslog
          sleep 2
        fi
      fi
    elif [ "$jucon" = n ];then
      echo "continue to run $0"
      break
    else
      echo 'print error'
      exit 10
    fi
  done
echo ""
fi

date=`date +%F_%H:%M:%S`

##3.日志清理策略,
##对一天以前生成的日志进行清理
dellog=`find /var/log/systatus/* -type f -mtime 1 -o -mtime +1 2>/dev/null`
if [ -n "$dellog" ];then
  for log in $dellog;do
    rm -rf $log
  done
echo ""
echo "Old systatus log clean finished"
echo "Old systatus log clean finished" >> $toolslog
fi

dellogt=`find /var/sysadmin_toolbox/other/systools/logs/* -type f -mtime +5 2>/dev/null`
if [ -n "$dellogt" ];then
  for logt in $dellogt;do
    rm -rf $logt
  done
echo ""
echo "Old systools run log clean finished"
echo "Old systools run log clean finished" >> $toolslog
fi

##5.输出检查时间:执行时间,系统版本,pid,ip的信息
##1).系统版本号,判断是suse还是redhat系统
if [ -f "/etc/redhat-release" ];then
  sysver=`cat /etc/redhat-release`
else
  sysver=`cat /etc/Suse-release | head -1`
fi

##2).输出系统基础信息
echo ""
echo -e "\033[1mbase information: \033[0m"
echo "check time: $date"
echo "run pid: $pid_lo"
echo "ip: `ifconfig -a | grep -v 127.0.0.1 | grep inet | grep -v inet6 | awk '{print $2}' | head -1`"
echo "system version: $sysver" 
echo "" 
echo -e "\033[1mInspection items: \033[0m"
sleep 2

lonum=1
swnum=1
sinum=1
finum=1
while [ "$lonum" -le "$lofre" ];do
##在循环执行的每一次执行前,判断如果日志文件大于20M,为了防止循环执行时脚本过大,退出脚本执行
if [ -f "$logname" ];then
  filesize=`du -s $logname | awk '{print $1}'`
  if [ "$filesize" -ge 20480 ];then
    echo "!!Systatus log is more than 20M!!"
    echo "Error: Systatus log is more than 20M,exit to run script" >> $toolslog
    exit 10
  fi
fi

##6.每个检查项情况
##1).cpu load情况
clo(){
##获取1分钟;5分钟;15分钟负载情况
load1=`uptime | awk -F"[, ]+" '{print $(NF-2)}'`
load5=`uptime | awk -F"[, ]+" '{print $(NF-1)}'`
load15=`uptime | awk -F"[, ]+" '{print $NF}'`

##获取逻辑cpu数量;物理cpu数量;cpu核数
log_cpu=`cat /proc/cpuinfo| grep "processor"| wc -l`
phy_cpu=`cat /proc/cpuinfo| grep "physical id"| sort| uniq| wc -l`
cpu_core=`cat /proc/cpuinfo| grep "cpu cores"| uniq | awk -F":" '{print $2}'`

##对cpuload数值进行计算，并与逻辑cpu个数进行比较
ave_load=`printf "%.2f" "$(echo "scale=2;((${load1}+${load5}+${load15})/3)" | bc)"`
load_com=`expr "$ave_load > $log_cpu" | bc`

echo -e "\033[1;35m-------------check cpu load------------\033[0m"
sleep 2
echo "###cpu load status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname
sleep 2
echo ""
echo "" >> $logname
#echo -e "\033[1;30;47m####cpu load status####\033[0m" 
##输出逻辑cpu个数及1,5,15min的cpu负载情况
echo "logical cpu num: $log_cpu"
echo "logical cpu num: $log_cpu" >> $logname
echo "cpu load status: 1m $load1, 5m $load5, 15m $load15"
echo "cpu load status: 1m $load1, 5m $load5, 15m $load15" >> $logname
echo "" 
echo ""  >> $logname
#sleep 2
}

##2).cpu使用率情况
cuse(){
echo -e "\033[1;35m-------------check cpu used------------\033[0m"
sleep 2
echo "###cpu used status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname
#sleep 2
echo ""
echo "" >> $logname

cpufile="$tempdir/cuse.tmp"
sarfile="$tempdir/sar.tmp"
echo "" > $cpufile
echo "" > $sarfile

##通过sar命令获取系统cpu使用率
sar 1 3 >> $sarfile
##cpu使用率的各项指标
cpu_user=`printf "%.1f" $(cat $sarfile | tail -1 | awk '{print $3}')` ##%user
cpu_nice=`printf "%.1f" $(cat $sarfile | tail -1 | awk '{print $4}')` ##%nice
cpu_sys=`printf "%.1f" $(cat $sarfile | tail -1 | awk '{print $5}')`  ##%sys
cpu_io=`printf "%.1f" $(cat $sarfile | tail -1 | awk '{print $6}')`   ##%iowait
cpu_st=`printf "%.1f" $(cat $sarfile | tail -1 | awk '{print $7}')`   ##%steal
cpu_used=`printf "%.1f" $(echo "scale=2;($cpu_user+$cpu_nice+$cpu_sys+$cpu_io+$cpu_st)" | bc)`

##对系统cpu使用率进行判断
##50%-60%;60%-70%;70%以上
if [ "`echo "$cpu_used >= 50" | bc`" -eq 1 ] && [ "`echo "$cpu_used < 60" | bc`" -eq 1 ];then
  cpu_level="!relatively high!"    ##50%=<cpu per used<60%
elif [ "`echo "$cpu_used >= 60" | bc`" -eq 1 ] && [ "`echo "$cpu_used < 70" | bc`" -eq 1 ];then
  cpu_level="!!high!!"        ##60%=<cpu per used<70%
elif [ "`echo "$cpu_used >= 70" | bc`" -eq 1 ];then
  cpu_level="!!!extremely high!!!"     ##70%=<cpu per used
else 
  cpu_level="normal"        ##cpu per used<50%
fi

##输出cpu使用率
echo "CPU usage(all): $cpu_used%" 
echo "CPU usage(all): $cpu_used%" >> $logname
echo "Composition: %user $cpu_user, %nice $cpu_nice, %sys $cpu_sys, %io $cpu_io %steal $cpu_st"
echo "Composition: %user $cpu_user, %nice $cpu_nice, %sys $cpu_sys, %io $cpu_io %steal $cpu_st" >> $logname
echo "" 
echo ""  >> $logname
echo "detection result: $cpu_level" 
echo "detection result: $cpu_level" >> $logname
echo "" 
echo "" >> $logname
sleep 2

##获取cpu使用率前10的线程,输出到临时文件中
#ps H -eo pid,tid,etimes,%cpu,comm --no-headers --sort -%cpu | head -10 >> $cpufile
ps H -eo pid,tid,stime,%cpu,comm --no-headers --sort -%cpu | head -10 >> $cpufile
echo "Top 10 cpu usage(current):"
echo "Top 10 cpu usage(current):" >> $logname
printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" "NAME(THREAD)" "TID" "PID" "STIME" "%CPU"
printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" "NAME(THREAD)" "TID" "PID" "STIME" "%CPU" >> $logname
##在cpu情况统计的临时文件中抓取出pid,tid,运行时间,cpu使用率,线程名称的信息,并进行输出
for i in `cat $cpufile | sed '/^$/d' | awk '{print $2}'`;do
  cline1=`awk -v a="$i" '$2==a {print $1}' $cpufile`  ##pid
  cline2=`awk -v a="$i" '$2==a {print $2}' $cpufile`  ##tid
  cline3=`awk -v a="$i" '$2==a {print $3}' $cpufile`  ##运行时间
  cline4=`awk -v a="$i" '$2==a {print $4}' $cpufile`  ##cpu使用率
  cline5=`awk -v a="$i" '$2==a {print $5}' $cpufile`  ##线程名称
  printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" $cline5 $cline2 $cline1 $cline3 $cline4%
  printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" $cline5 $cline2 $cline1 $cline3 $cline4% >> $logname
#  sleep 1
done
echo ""
echo "" >> $logname
#sleep 2
}


##3).内存使用率情况
muse(){
freefile="$tempdir/free.tmp"
memfile="$tempdir/psmem.tmp"
echo "" > $freefile
echo "" > $memfile

echo -e "\033[1;35m-----------check memory used-----------\033[0m"
sleep 2
echo "###memory used status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname
sleep 1
echo "" 
echo ""  >> $logname

##通过free命令抓取出内存数据输入到临时文件中
##按照监控的计算方式进行取值,目前监控的已使用内存计算方式是free命令的used数值加上meminfo文件
##中的shmem数值
        free -k >> $freefile
##获取内存使用率排名前10的线程输出到临时文件中
        ps -eo pid,stime,%mem,comm --no-headers --sort -%mem | head -10 >> $memfile
        grep "cache:" $freefile > /dev/null
        if [[ $? -eq 0 ]];then   ##centos6;suse计算方式
                usedMem1=`cat $freefile|grep "cache:" |awk '{print $3}'`
                freeMem=`cat $freefile|grep "cache:" |awk '{print $4}'`
                shmem=`cat /proc/meminfo |grep -w Shmem|awk '{print $2}'`
                usedMem=`expr $usedMem1 + $shmem`
        else    ##redhat7计算方式
                usedMem1=`cat $freefile|grep "Mem:" |awk '{print $3}'`  
                freeMem=`cat $freefile|grep "Mem:" |awk '{print $4}'`   
                shmem=`cat /proc/meminfo |grep -w Shmem|awk '{print $2}'`   
                usedMem=`expr $usedMem1 + $shmem`      ##内存占用used+shmem
        fi
        totalMem=`cat $freefile|grep "Mem:" |awk '{print $2}'`
        usedMemPct=`expr $usedMem \* 100 / $totalMem `
        freeMemPct=`expr 100 - $usedMemPct `

if [ "$usedMemPct" -ge 50 ] && [ "$usedMemPct" -lt 60 ];then
  mem_level="!relatively high!"    ##50%=<mem per used<60%
elif [ "$usedMemPct" -ge 60 ] && [ "$usedMemPct" -lt 70 ];then
  mem_level="!!high!!"  ##60%=<mem per used<70%
elif [ "$usedMemPct" -ge 70 ];then
  mem_level="!!!extremely high!!!"  ##70%=<mem per used
else
  mem_level="normal"    ##mem per used<50%
fi

echo "memory usage: ${usedMemPct}%" 
echo "memory usage: ${usedMemPct}%"  >> $logname 
echo "detection result: $mem_level" 
echo "detection result: $mem_level"  >> $logname 
echo ""  
echo ""  >> $logname
sleep 2

echo "Top 10 memory usage:" 
echo "Top 10 memory usage:"  >> $logname
printf "%-20s %-10s %-10s %-10s %-10s\n" "NAME(process)" "PID" "STIME" "%MEM"
printf "%-20s %-10s %-10s %-10s %-10s\n" "NAME(process)" "PID" "STIME" "%MEM" >> $logname
for i in `cat $memfile | sed '/^$/d' | awk '{print $1}'`;do
  mline1=`awk -v a="$i" '$1==a {print $1}' $memfile`  ##pid
  mline2=`awk -v a="$i" '$1==a {print $2}' $memfile`  ##运行时间
  mline3=`awk -v a="$i" '$1==a {print $3}' $memfile`  ##内存使用率
  mline4=`awk -v a="$i" '$1==a {print $4}' $memfile`  ##进程名称
  printf "%-20s %-10s %-10s %-10s %-10s\n" $mline4 $mline1 $mline2 $mline3%
  printf "%-20s %-10s %-10s %-10s %-10s\n" $mline4 $mline1 $mline2 $mline3% >> $logname
#  sleep 1
done
echo ""
echo "" >> $logname
#sleep 2
}

##swap分区使用率 
swu(){
##swap利用率执行次数
swapfile="$tempdir/swa.tmp"
pswapfile="$tempdir/pswa.tmp"
echo -e "\033[1;35m------------check swap used------------\033[0m"
sleep 2
echo "###swap used status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname
sleep 2
echo ""
echo "" >> $logname

echo "" > $swapfile
echo "" > $pswapfile

##swap分区总大小、已使用
free -k >> $swapfile
swato=`cat $swapfile | tail -1 | awk '{print $2}'` ##swap总空间大小
if [ "$swato" != 0 ];then
  swaused=`cat $swapfile | tail -1 | awk '{print $3}'`  #swap已使用大小
  swapuser=`printf "%.1f" "$(echo "scale=3;($swaused/$swato)*100" | bc)" 2>/dev/null`  ##swap利用率
  vmst=`vmstat`
  swin=`echo "$vmst" | tail -1 | awk '{print $7}'`
  swout=`echo "$vmst" | tail -1 | awk '{print $8}'`
fi

##判断swap分区是否开启
if [ "$swato" = 0 ];then
  swapsta="--close--"
  swstan=0
else
  swapsta="open"
  swstan=1
fi

echo "swap status: $swapsta"
if [ "$swstan" = 0 ];then
  echo ""
  echo "" >> $logname
  return 1
else
  echo "swap total: `free -h | tail -1 | awk '{print $2}'`" 
  echo "swap total: `free -h | tail -1 | awk '{print $2}'`"  >> $logname
  echo "swap used: $swapuser%"
  echo "swap used: $swapuser%" >> $logname
  if [ "`echo "$swapuser >= 20" | bc`" -eq 1 ] && [ "`echo "$swapuser < 30" | bc`" -eq 1 ];then  ##swap占用率大于20%
    swcheck="!relatively high!"  
  elif [ "`echo "$swapuser >= 30" | bc`" -eq 1 ] && [ "`echo "$swapuser < 40" | bc`" -eq 1 ];then  ##swap占用率大于30%
    swcheck="!!high!!"
  elif [ "`echo "$swapuser >= 40" | bc`" -eq 1 ];then    ##swap占用率大于40%
    swcheck="!!!extremely high!!!"
  else
    swcheck="normal"    
  fi

  echo "detection result: $swcheck"
  echo "detection result: $swcheck" >> $logname
  echo "" 
  echo "" >> $logname
  echo "swap in(kb/s) $swin, swap out(kb/s) $swout"
  echo "swap in(kb/s) $swin, swap out(kb/s) $swout" >> $logname
  echo "" 
  echo "" >> $logname
fi

if [ "$swapuser" = '0.0' ];then
  echo ""
  echo "" >> $logname
  return 1
fi

sleep 2

##判断时间间隔,距离上次执行小于30min,不执行获取每个进程占用swap分区的状态
##当前时间戳
if [ "$swnum" -gt 1 ];then
  curstm=`date -d "$(date)" +%s`
  difstm=`printf "%.0f" $(scale=1;echo "$curstm-$oldsta" | bc)`
  if [ "`echo "$difstm >= 1800" | bc`" = 0 ];then 
    echo "process swap usage check time interval less than 30min in last check"
    echo "process swap usage check time interval less than 30min in last check" >> $logname
    echo ""
    echo "" >> $logname
    return 1
  fi
fi

##进程占用swap分区的百分比
getswap(){
SUM=0
swato=`free -m | tail -1 | awk '{print $2}'`
for DIR in `find /proc/ -maxdepth 1 -type d | egrep "^/proc/[0-9]"`;do
    PID=`echo $DIR | cut -d / -f 3`
    PROGNAME=`ps -p $PID -o comm --no-headers`
    for SWAP in `grep Swap $DIR/smaps 2>/dev/null| awk '{ print $2 }'`;do
        let SUM=$SUM+$SWAP
#        sleep 2
    done
       SUM_M=`printf "%.1f" "$(echo "scale=4;($SUM/1024)" | bc)"`
       swrtime=`ps -A -o pid,stime | awk -v a="$PID" '$1==a {print $2}'`
    printf "%-20s %-10s %-10s %-13s %-15s\n" "$PROGNAME" "$PID" "$swrtime" "${SUM_M}M" `printf "%.1f" "$(echo "scale=4;($SUM_M/$swato)*100" | bc)"`"%"
    SUM=0
#    sleep 1
done
}
swexc=`getswap | sort -nr -k4 | head -10`

echo "Top 10 in swap usage:"  
echo "excute num: $swnum"
echo "Top 10 in swap usage:"  >> $logname
echo "excute num: $swnum"  >> $logname
printf "%-20s %-10s %-10s %-10s %-15s\n" "NAME(PROCESS)" "PID" "STIME" "USED_VALUE(M)" "USED(%)" 
printf "%-20s %-10s %-10s %-10s %-15s\n" "NAME(PROCESS)" "PID" "STIME" "USED_VALUE(M)" "USED(%)" >> $logname
echo "$swexc"
echo "$swexc" >> $logname
echo ""    
echo ""  >> $logname
oldsta=`date -d "$(date)" +%s`
let swnum++
}

##进程D状态信息
psdr(){
psdfile="$tempdir/psd.tmp"
psrfile="$tempdir/psr.tmp"
echo "" > $psdfile
echo "" > $psrfile

echo -e "\033[1;35m-----check process in D&&R status------\033[0m"
sleep 2
echo "###process in D&&R status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname 
sleep 2
echo ""
echo "" >> $logname

##获取D状态进程
statd(){
  numd=0
  for pidd in `ls /proc | grep "^[0-9]"`;do
    if [ "$pidd" -gt 0 ] 2>/dev/null;then
      rund=`cat /proc/$pidd/status 2>/dev/null | grep "disk sleep" | wc -l`
      if [ "$rund" -gt 0 ] 2>/dev/null;then           
        runtimed=`ps -A -o pid,stime | awk -v var1=$pidd '$1==var1{print $2}'`
        tasknamed=`cat /proc/$pidd/status 2>/dev/null| grep Name | awk -F" " '{print $2}'`
        wchand=`cat /proc/$pidd/wchan`
        if [[ "$wchand" != "" ]] && [[ "$runtimed" != "" ]] && [[ "$tasknamed" != "" ]];then
          numd=$(($numd+1))
          printf "%-10s %-20s %-15s %-15s\n" "$pidd" "$tasknamed" "$runtimed" "$wchand" >> $psdfile
        else
          continue
        fi
      fi
    fi
#sleep 1
done
echo "dnum:$numd" >> $psdfile
}
statd

tonumd=`grep "dnum"  $psdfile | awk -F":" '{print $2}'`
echo "---process in D state---"
echo "---process in D state---" >> $logname
echo "D state number: $tonumd"
echo "D state number: $tonumd" >> $logname
if [ "$tonumd" = 0 ];then
  echo "detection result: not exist"
  echo "detection result: not exist" >> $logname
else
  echo "detection result: !!exist!!"
  echo "detection result: !!exist!!" >> $logname
fi
echo ""
echo "" >> $logname

if [ "$tonumd" -gt 0 ];then
  echo "PROCESS DETAIL(D):"
  printf "%-10s %-20s %-15s %-15s\n" "PID" "NAME" "STIME" "WCHAN" 
  printf "%-10s %-20s %-15s %-15s\n" "PID" "NAME" "STIME" "WCHAN" >> $logname
  cat $psdfile | grep -v "dnum" | sed '/^$/d'
  cat $psdfile | grep -v "dnum" | sed '/^$/d' >> $logname
fi
echo ""
echo "" >> $logname
sleep 2

##获取R状态进程
statr(){
rnum=0
for pidr in `ls /proc`;do
  if [ "$pidr" -gt 0 ] 2>/dev/null;then
    runr=`cat /proc/$pidr/status 2>/dev/null| grep "running" | wc -l`
    if [ "$runr" -gt 0 ] 2>/dev/null;then
      runtimer=`ps -A -o pid,stime | awk -v var1=$pidr '$1==var1{print $2}'`
      tasknamer="`cat /proc/$pidr/status 2>/dev/null | grep Name | awk -F" " '{print $2}'`"
      wchanr=`cat /proc/$pidr/wchan`
      if [[ "$runtimer" != "" ]] && [[ "$tasknamer" != "" ]] && [[ "$wchanr" != "" ]];then
        rnum=$(($rnum+1))
        printf "%-10s %-20s %-15s %-15s\n" "$pidr" "$tasknamer" "$runtimer" "$wchanr" >> $psrfile
      fi
    fi
  fi
#  sleep 1
done
echo "rnum:$rnum" >> $psrfile
}
statr

tonumr=`grep "rnum" $psrfile | awk -F":" '{print $2}'`
echo "---process in R state---"
echo "---process in R state---" >> $logname
echo "R state number: $tonumr"
echo "R state number: $tonumr" >> $logname
if [ "$tonumr" -gt 0 ];then
  echo ""
  echo "" >> $logname
  echo "PROCESS DETAIL(R):"
  echo "PROCESS DETAIL(R):" >> $logname
  printf "%-10s %-20s %-15s %-15s\n" "PID" "NAME" "STIME" "WCHAN"
  printf "%-10s %-20s %-15s %-15s\n" "PID" "NAME" "STIME" "WCHAN" >> $logname
  cat $psrfile | grep -v "rnum" | sed '/^$/d'
  cat $psrfile | grep -v "rnum" | sed '/^$/d' >> $logname
fi
echo ""
echo "" >> $logname
#sleep 2
}

##inode号
ino(){
echo -e "\033[1;35m------------check inode used-----------\033[0m"
sleep 2
echo "###inode used status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname
sleep 2
echo ""
echo "" >> $logname
inodefile="$tempdir/ino.tmp"
echo "" > $inodefile
df -i | awk 'NR>2{print line}{line=$0} END{print line}' | sort -nr -k5 | head -3 >> $inodefile

echo "inode usage(top 3):"
echo "inode usage(top 3):" >> $logname
printf "%-20s %-10s\n" "MOUNT" "USED(%)"
printf "%-20s %-10s\n" "MOUNT" "USED(%)" >> $logname
for i in `cat $inodefile | awk '{print $6}'`;do
  inodeuse=`awk -v a="$i" '$6==a{print $5}' $inodefile`
  printf "%-20s %-10s\n" $i $inodeuse
  printf "%-20s %-10s\n" $i $inodeuse >> $logname
  sleep 1
done
echo ""
echo "" >> $logname
sleep 2
}

##获取占用空间前3的目录
##获取到2级目录下空间大小占用前3的目录
size(){
echo -e "\033[1;35m------check directory size status------\033[0m"
echo "###directory size status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname 
sleep 2
echo ""
echo "" >> $logname

sizetmp="$tempdir/size.tmp"

##判断执行时间是否间隔30min
if [ "$lonum" -gt 1 ];then
  cursiti=`date -d "$(date)" +%s`
  difsiva=`printf "%.0f" $(scale=1;echo "$cursiti-$oldstasi" | bc)`
  if [ "`echo "$difsiva>=1800" | bc`" = 0 ];then
    echo "directory size status check time interval less than 30min in last check"
    echo "directory size status check time interval less than 30min in last check" >> $logname
    echo ""
    echo "" >> $logname
    return 1
  fi
fi

echo "" > $sizetmp
##/下空间占用排名前3的目录
du -sh /* 2>/dev/null | sort -rh -k1 | head -3 >> $sizetmp
echo "directory space usage(top 3,secondary directory):" 
echo "excute num: $sinum"
echo "directory space usage(top 3,secondary directory):" >> $logname
echo "excute num: $sinum"  >> $logname
for i in `cat $sizetmp | awk '{print $2}'`;do
  ##获取/下占用空间前3的目录排序
  sizechil=`du -sh $i/* | sort -rh -k1 | head -3` 
  sizet=`grep -w $i $sizetmp | awk '{print $1}'`
  printf "%-10s %-10s\n" "$i" "$sizet" 
  printf "%-10s %-10s\n" "$i" "$sizet" >> $logname
  sleep 1
  ##获取/下占用空间前3的目录下，占用空间前3的二级目录
  for j in `echo "$sizechil" | awk '{print $2}'`;do
    sizec=`echo "$sizechil" | grep -w "$j" | awk '{print $1}'`
    printf "%-25s %-10s\n" " --- $j" "$sizec" 
    printf "%-25s %-10s\n" " --- $j" "$sizec" >> $logname
  done
  echo ""
  echo "" >> $logname
sleep 1
done
let sinum++
oldstasi=`date -d "$(date)" +%s`
}

##系统进程io情况
iot(){
echo -e "\033[1;35m---------check system io status--------\033[0m"
sleep 2
echo "###system io status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname 
sleep 2
echo ""
echo "" >> $logname

##生成的临时文件
diskio="$tempdir/diskio.tmp"
proio="$tempdir/proio.tmp"
echo '' > $diskio
echo '' > $proio

echo "---disk io status---"
echo "---disk io status---" >> $logname
##磁盘io情况
diskname=`lsblk | grep '^[a-z]d[a-z]' | awk '{print $1}'`
iow=`iostat -x | grep -A 1 'avg-cpu' | awk 'NR==2{print $4}'`
iostat -x >> $diskio
echo "system iowait(%): $iow"
echo "system iowait(%): $iow" >> $logname
echo ""
echo "" >> $logname
sleep 2

##磁盘io情况
echo "disk usage io in detail"
echo "disk usage io in detail" >> $logname
printf "%-10s %-10s %-10s %-10s %-10s\n" "DISK" "TPS" "AWAIT(ms)" "SVCTM(ms)" "UTIL(%)"
printf "%-10s %-10s %-10s %-10s %-10s\n" "DISK" "TPS" "AWAIT(ms)" "SVCTM(ms)" "UTIL(%)" >> $logname
for i in $diskname;do
  ##tps
  tpsio=`iostat | grep -w "^$i" | awk '{print $2}'`
  ##await
  waitio=`cat $diskio | grep -w "^$i" | awk '{print $10}'`
  ##svctm
  ctmio=`cat $diskio | grep -w "^$i" | awk '{print $13}'`
  ##util
  utiio=`cat $diskio | grep -w "^$i" | awk '{print $14}'`
  printf "%-10s %-10s %-10s %-10s %-10s\n" "$i" "$tpsio" "$waitio" "$ctmio" "$utiio"
  printf "%-10s %-10s %-10s %-10s %-10s\n" "$i" "$tpsio" "$waitio" "$ctmio" "$utiio" >> $logname
  sleep 1
done
sleep 1

echo ""
echo "" >> $logname
echo "---process io status---"
echo "---process io status---" >> $logname
##进程io情况
pidstat -d 1 3 | grep -i 'Average' | grep -v 'UID' >> $proio
pionum=`cat $proio | sed '/^$/d' | wc -l`
if [ "$pionum" = 0 ];then
  echo "process use io is not exist"
  echo "process use io is not exist" >> $logname
else
  echo "process io usage in detail(current):"
  echo "process io usage in detail(current):" >> $logname
  printf "%-15s %-10s %-10s %-10s %-10s %-10s\n" "NAME" "PID" "STIME" "IO_R(KB/S)" "IO_W(KB/S)" "IO_CCWR(KB/S)"
  printf "%-15s %-10s %-10s %-10s %-10s %-10s\n" "NAME" "PID" "STIME" "IO_R(KB/S)" "IO_W(KB/S)" "IO_CCWR(KB/S)" >> $logname
  pion=0
  for j in `cat $proio | awk '{print $3}'`;do
    iona=`cat $proio | awk -v a="$j" '$3==a {print $7}'`
    iopr=`cat $proio | awk -v a="$j" '$3==a {print $4}'`
    iopw=`cat $proio | awk -v a="$j" '$3==a {print $5}'`
    ioccw=`cat $proio | awk -v a="$j" '$3==a {print $6}'`
    iotime=`ps -A -o pid,stime | awk -v a="$j" '$1==a {print $2}'`
    printf "%-15s %-10s %-10s %-10s %-10s %-10s\n" "$iona" "$j"  "$iotime" "$iopr" "$iopw" "$ioccw"
    printf "%-15s %-10s %-10s %-10s %-10s %-10s\n" "$iona" "$j"  "$iotime" "$iopr" "$iopw" "$ioccw" >> $logname
    let pion++
  #  sleep 1
  done
  
  echo "use io process num(current): $pion"
  echo "use io process num(current): $pion" >> $logname
fi
echo ""
echo "" >> $logname
#sleep 2
}


##文件描述符
fde(){
echo -e "\033[1;35m---------check system fd status--------\033[0m"
sleep 2
echo "###system fd status###" >> $logname
echo "time: `date +%F_%H:%M:%S`"
echo "time: `date +%F_%H:%M:%S`" >> $logname 
sleep 2
echo ""
echo "" >> $logname

##判断执行时间间隔是否大于30min
if [ "$lonum" -gt 1 ];then
  fdcurti=`date -d "$(date)" +%s`
  fddifti=`printf "%.0f" $(scale=1;echo "$fdcurti-$fdotime" | bc)`
  if [ "`echo "$fddifti>=1800" | bc`" = 0  ];then
    echo "system fd check time interval less than 30min in last check"
    echo "system fd check time interval less than 30min in last check" >> $logname
    echo ""
    echo "" >> $logname
  return 1
  fi
fi
echo "excute num: $finum"
fdtmp="$tempdir/fdfile.tmp"
echo '' > $fdtmp
##基本信息
##配置的文件最大打开数,单个用户最大文件打开数
fileto=`cat /proc/sys/fs/file-max`
userfito=`ulimit -n`
echo "max open file number in file(system config): $fileto"
echo "max open file number(user config): $userfito"
echo ""
echo "max open number in file(system config): $fileto" >> $logname
echo "max open number in file(user config): $userfito"  >> $logname
echo "" >> $logname
sleep 2
sysps=`ps -A -o pid,user,stime,comm`
##打开文件总数
lsof -n | grep -v "^COMMAND.*PID" >> $fdtmp
filenum=`cat $fdtmp | sed '/^$/d' | wc -l`

##计算当前进程打开的文件数量情况
fiperuse=`printf "%.2f\n" $(echo "scale=5;($filenum/$fileto)*100" | bc)`
echo "file open number(current): $filenum"
echo "file open used(%): ${fiperuse}%"
echo ""
echo "file open number(current): $filenum" >> $logname
echo "file open used(%): ${fiperuse}%" >> $logname
echo "" >> $logname
sleep 2

##输出打开文件前10的进程
filepro=`cat $fdtmp | sed '/^$/d' | awk '{ A[$2]++ } END{for (B in A){ print B, A[B] } }'| sort -nr -k2`
echo "process open file number in Top 10:"
echo "process open file number in Top 10:" >> $logname
printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" "NAME" "USER" "PID" "STIME" "OPENNUM" "USED(%)"
printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" "NAME" "USER" "PID" "STIME" "OPENNUM" "USED(%)" >> $logname
for i in `echo "$filepro" | awk '{print $1}'`;do
  ##进程名称
  #fpsname=`ps -A -o pid,comm | awk -v a=$i '$1==a{print $2}'`
  fpsname=`echo "$sysps" | awk -v a=$i '$1==a{print $4}'`
  ##进程运行时间
  #fpstim=`ps -A -o pid,stime | awk -v a=$i '$1==a{print $2}'`
  fpstim=`echo "$sysps" | awk -v a=$i '$1==a{print $3}'`
  ##进程运行用户
  fpuser=`echo "$sysps" | awk -v a=$i '$1==a{print $2}'`
  ##进程文件打开数
  psfinum=`echo "$filepro" | awk -v a=$i '$1==a{print $2}'`
  ##进程文件打开数占比
  psperfi=`printf "%.2f\n" $(echo "scale=5;($psfinum/$fileto)*100" | bc)`
  ##输出所有信息
  printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" "$fpsname" "$fpuser" "$i" "$fpstim" "$psfinum" "$psperfi%"
  printf "%-20s %-10s %-10s %-10s %-10s %-10s\n" "$fpsname" "$fpuser" "$i" "$fpstim" "$psfinum" "$psperfi%" >> $logname
done | head -10
echo ""
echo "" >> $logname
fdotime=`date -d "$(date)" +%s`
}

$1 2>/dev/null
if [ "$1" = "all" ];then
  clo  
  cuse
  muse
  swu
  psdr
  ino
  size
  iot
  fde
fi
$2 2>/dev/null
$3 2>/dev/null
$4 2>/dev/null
$5 2>/dev/null
$6 2>/dev/null
$7 2>/dev/null
$8 2>/dev/null
$9 2>/dev/null
$10 2>/dev/null 
$11 2>/dev/null
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo -e "\033[1;32mexecution completed number: $lonum\033[0m"
echo "------execution completed number: $lonum------" >> $logname
echo ""
echo "" >> $logname
if [ -n "$lofre" ];then
  if [ "$lofre" != "$lonum" ];then
    sleep $extime
  fi
fi
let lonum++
done

finlonum=$(($lonum-1))

trap "rm -rf $tempdir && 
      echo 'Systools execution completed,time: `date +%F_%H:%M:%S` (pid: $pid_lo)' >> $toolslog &&
      echo 'number of runs: $finlonum (pid: $pid_lo)'  >> $toolslog &&
      echo 'systatus log location: $logname (pid: $pid_lo)' >> $toolslog &&
      echo 'pid: $pid_lo number of runs in this time:' $finlonum &&
      echo 'systatus log file:' `echo $logname` &&
      exit 10" EXIT

echo -e "\033[1;32m##############system status check finish##############\033[0m"