#!/bin/bash
#
# StackX Monitoring System (SMS)
# sms.sh / sms.sh
# Author: Christophe Casalegno / Brain 0verride
# Contact: brain@christophe-casalegno.com
# Version 1.0.3
#
# Copyright (c) 2020 Christophe Casalegno
#
# This program is free software: you can redistribute it and/or modify
#
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see
#
# The license is available on this server here:
# https://www.christophe-casalegno.com/licences/gpl-3.0.txt
#
# To inlude in a php page for example: (dont forget to put your script in a
# directory not readable by your web server)
#
# The graphic banner is available here:
# https://www.christophe-casalegno.com/banners/ScalarX.jpg
#
#
#
# Example for a CONF_FILE: (with only needed variable for this script)
# loginsxmysql:sx
# passsxmysql:prxotpoofas
#
# Example for process2monitor.txt (generated automatically by my deployer for each server)
# apache2
# fail2ban-server
# memcached
# miniserv.pl
# php-fpm5.6
# php-fpm7.0
# php-fpm7.1
# php-fpm7.2
# php-fpm7.3
# php-fpm7.4
# php-fpm8.0
# pure-ftpd
# sshd
#
# Example for fs_list.txt (generated automatically by my deployer for each server)
#
# grep -v '#' /etc/fstab |awk '{print $2}' > fs_list.txt
#
# If you use other filesystem like network sshfs you may add:
# grep 'sshfs' /etc/fstab |awk '{print $2}' >> fs_list.txt
#
# Result can be something like :
# /
# /datastore
# /datadrop/cd101
# etc. depending of your configuration
CONF_FILE="/home/sx/.sx" # Replace by your config file
PROCESS_CONF="/home/sx/data/process2monitor.txt"
FILE2CHECK=$(cat $PROCESS_CONF)
function read_config()
{
CONF_FILE="$1"
VAR_CONF=$(cat $CONF_FILE |sed "s/ /_/g")
for LINE in $VAR_CONF
do
VARNAME1=${LINE%%:*}
VARNAME2=${VARNAME1^^}
VAR=${LINE#*:}
eval ${VARNAME2}=$VAR
# Alternative with external programs like cut, grep and tr
# VARNAME=$(echo $LINE |cut -d ":" -f1 |tr [:lower:] [:upper:])
# VAR=$(echo $LINE |grep -w "$VAR_CONF" |cut -d ":" -f2)
# eval ${VARNAME}=$VAR
done
}
read_config $CONF_FILE
function format()
{
TARGETFORMAT="$1"
CHAIN2FORMAT="$2"
if [[ "$TARGETFORMAT" = 'N' ]]
then
echo "$CHAIN2FORMAT"
elif [[ "$TARGETFORMAT" = 'O' ]]
then
echo "$CHAIN2FORMAT"
elif [[ "$TARGETFORMAT" = 'W' ]]
then
echo "$CHAIN2FORMAT"
elif [[ "$TARGETFORMAT" = 'E' ]]
then
echo "$CHAIN2FORMAT"
else
echo 'format not specified'
fi
}
function checktest()
{
if [ "$1" -eq 0 ]
then
format N "$2:"
format O "OK"
else
format N "$2:"
format E "ERROR"
fi
echo '
'
}
function checkwarning()
{
if [ "$1" -eq 0 ]
then
format N "$2:"
format O "OK"
else
format N "$2:"
format W "WARNING"
fi
echo '
'
}
function startpage()
{
HTML_TITLE="$1"
echo ''
echo '
'
echo "$HTML_TITLE"
echo ''
echo ''
}
function title()
{
TITLE="$1"
echo ''
echo ''
echo ' | | '
echo "$TITLE"
echo ' |
'
echo '
'
}
function titlecheck()
{
TITLE_CHECK="$1"
echo "
"
format N "$TITLE_CHECK"
echo "
"
}
function table_init()
{
echo ''
}
function center_column()
{
echo ' | '
echo ' | '
echo ''
}
function table_end()
{
echo ' |
'
}
function endpage()
{
echo "";
echo "";
}
function all_process_check()
{
for LINE2CHECK in $FILE2CHECK
do
check_process "$LINE2CHECK"
echo "
"
done
}
function check_process()
{
P2CHECK="$1"
PROCESS=$(pgrep -c "$P2CHECK")
if [[ "$PROCESS" -eq 0 ]]
then
format N "$P2CHECK:"
format E "ERROR"
else
format N "$P2CHECK:"
format O "OK"
fi
}
function check_update()
{
LAST_UPDATE=$(grep "Start-Date" /var/log/apt/history.log |tail -n 1 |cut -d " " -f2)
LAST_DATE=$(date -d "$LAST_UPDATE" +%s)
NOW_DATE=$(date +%s)
DAYS_ALERT=$1
DIFF_DAYS=$(( (NOW_DATE - LAST_DATE) / 86400 ))
echo '
'
titlecheck "System updates"
if [[ $DIFF_DAYS -gt 0 && $DIFF_DAYS -gt $DAYS_ALERT ]]
then
format N "Last apt updates ($LAST_UPDATE):"
format W "WARNING"
echo '
'
else
format N "Last apt updates ($LAST_UPDATE):"
format O "OK"
fi
}
function disk_check()
{
SPACE_ERROR_TRESHOLD="$1"
SPACE_WARNING_TRESHOLD="$2"
INODES_ERROR_TRESHOLD="$3"
INODES_WARNING_TRESHOLD="$4"
titlecheck "Disks"
LIST_DISK=$(df -x tmpfs -x devtmpfs -x squashfs | grep 'dev' |awk -F " " '{print $1}' |cut -d/ -f3)
for DISK in $LIST_DISK
do
SPACE_USED_PERCENT=$(df |grep -w "$DISK" |head -1|awk -F" " '{print $5}' |cut -d% -f1)
SPACE_USED=$(df -x tmpfs -x devtmpfs -x squashfs -h |grep "$DISK" |head -1 |awk -F " " '{print $3}')
SPACE_TOTAL=$(df -x tmpfs -x devtmpfs -x squashfs -h |grep "$DISK" |head -1 |awk -F " " '{print $2}')
if [[ "$SPACE_USED_PERCENT" -gt "$SPACE_ERROR_TRESHOLD" ]]
then
format N "Part: $DISK - $SPACE_USED / $SPACE_TOTAL ($SPACE_USED_PERCENT%) space:"
format E "ERROR"
echo '
'
elif [[ "$SPACE_USED_PERCENT" -gt "$SPACE_WARNING_TRESHOLD" ]]
then
format N "Part: $DISK - $SPACE_USED / $SPACE_TOTAL ($SPACE_USED_PERCENT%) space:"
format W "WARNING"
echo '
'
else
format N "Part: $DISK - $SPACE_USED / $SPACE_TOTAL ($SPACE_USED_PERCENT%) space:"
format O "OK"
echo '
'
fi
INODES_USED_PERCENT=$(df -i|grep -w "$DISK" |head -1 |awk -F" " '{print $5}' |cut -d% -f1)
INODES_USED=$(df -i -h |grep -w "$DISK" |head -1 |awk -F" " '{print $3}' |cut -d% -f1)
INODES_TOTAL=$(df -i -h |grep -w "$DISK" |head -1 |awk -F" " '{print $2}')
if [[ "$INODES_USED_PERCENT" -gt "$INODES_ERROR_TRESHOLD" ]]
then
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL ($INODES_USED_PERCENT%) inodes:"
format E "ERROR"
echo '
'
elif [[ "$INODES_USED_PERCENT" -gt "$INODES_WARNING_TRESHOLD" ]]
then
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL ($INODES_USED_PERCENT%) inodes:"
format W "WARNING"
echo '
'
else
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL ($INODES_USED_PERCENT%) inodes:"
format O "OK"
echo '
'
fi
done
}
function dns_check()
{
titlecheck "DNS resolution"
for DNS in "$@"
do
CHECK_DNS_RESOLUTION=$(dig +time=0 +tries=1 $DNS > /dev/null; echo "$?")
checkwarning "$CHECK_DNS_RESOLUTION" "$DNS"
done
}
function elastic_check()
{
IP="$1"
PORT="$2"
titlecheck "Elastic Search"
ELASTIC=$(curl -X GET "$IP:$PORT/" > /dev/null; echo "$?")
checktest "$ELASTIC" "Elastic Search connect (port 9200)"
}
function fs_check()
{
FS_ACTIVE="$1"
if [[ $FS_ACTIVE = "yes" ]]
then
titlecheck "Filesystems"
for LINE in $(cat /home/sx/data/fs_list.txt)
do
FS=$(df -x tmpfs -x devtmpfs -x squashfs |grep '\|@' |grep $LINE)
if [[ -z "$FS" ]]
then
format N "$LINE:"
format E "ERROR"
echo '
'
else
format N "$LINE:"
format O "OK"
echo '
'
fi
done
else
true
fi
}
function internet_check()
{
titlecheck "Internet"
for DNS in "$@"
do
CHECK_DNS_PING=$(ping -c 1 -W 1 $DNS > /dev/null; echo "$?")
checkwarning "$CHECK_DNS_PING" "$DNS ping"
done
}
function load_check()
{
THREADS=$(grep processor /proc/cpuinfo |wc -l)
LOAD_TRESHOLD=$(echo $(($THREADS * 4)))
WAIT_B4_CHECK="1"
echo "
"
titlecheck "Load Average"
LOAD_AVERAGE1=$(awk '{print $1}' < /proc/loadavg |cut -d "." -f1)
sleep "$WAIT_B4_CHECK"
LOAD_AVERAGE2=$(awk '{print $1}' < /proc/loadavg |cut -d "." -f1)
if [[ "$LOAD_AVERAGE1" -ge "$LOAD_TRESHOLD" ]]
then
if [[ "$LOAD_AVERAGE2" -ge "$LOAD_AVERAGE1" ]]
then
format N "Load average ($LOAD_AVERAGE2 / $LOAD_TRESHOLD) / $THREADS core(s):"
format E "ERROR"
else
format N "Load average ($LOAD_AVERAGE2 / $LOAD_TRESHOLD) / $THREADS core(s) but going down:"
format W "WARNING"
fi
else
format N "Load average ($LOAD_AVERAGE2 / $LOAD_TRESHOLD) / $THREADS core(s):"
format O "OK"
fi
}
function memcached_check()
{
IP="$1"
PORT="$2"
titlecheck "Memcached"
MEMCACHED=$(echo stats |timeout 1 bash -c " /dev/null; echo "$?")
checktest "$MEMCACHED" "Memcached connect (port 11211)"
}
function mem_check()
{
MEM_ERROR_TRESHOLD="$1"
MEM_WARNING_TRESHOLD="$2"
MEM_TOTAL=$(free -h |grep Mem |awk '{print $2}' |cut -d "i" -f1)
titlecheck "Memory"
MEM_USED_PERCENT=$(free |awk 'FNR == 2 {print 100-(($2-$3)/$2)*100}' |cut -d "." -f1)
MEM_USED=$(free -h |grep Mem |awk '{print $3}' |cut -d "i" -f1)
if [[ "$MEM_USED_PERCENT" -gt "$MEM_ERROR_TRESHOLD" ]]
then
format N "Memory usage: $MEM_USED / $MEM_TOTAL ($MEM_USED_PERCENT%):"
format E "ERROR"
elif [[ "$MEM_USED_PERCENT" -gt "$MEM_WARNING_TRESHOLD" ]]
then
format N "Memory usage: $MEM_USED / $MEM_TOTAL ($MEM_USED_PERCENT%):"
format W "WARNING"
else
format N "Memory usage: $MEM_USED / $MEM_TOTAL ($MEM_USED_PERCENT%):"
format O "OK"
fi
}
function mysql_check()
{
LOGINSXSQL="$1"
DBPASSWORDSQL="$2"
REPLICATION_TRESHOLD="$3"
DBSXSQL="sx"
TABLE_NAME="sx_test"
ERRORS=()
CONNECT="mysql -u$LOGINSXSQL --database=$DBSXSQL -p$DBPASSWORDSQL -e "
titlecheck "MySQL / MariaDB"
CONNECTION_SQL=$($CONNECT "SHOW VARIABLES LIKE '%version%';" > /dev/null; echo "$?")
checktest "$CONNECTION_SQL" "Connexion SQL"
#CREATE_TABLE=$($CONNECT "CREATE TABLE $TABLE_NAME(test varchar(255));" >/dev/null; echo "$?")
#checktest "$CREATE_TABLE" "Create table"
SHOW_TABLE=$($CONNECT "SHOW TABLES;" > /dev/null; echo "$?")
checktest "$SHOW_TABLE" "Show tables"
#DELETE_TABLE=$($CONNECT "DROP TABLE $TABLE_NAME;" >/dev/null; echo "$?")
#checktest "$DELETE_TABLE" "Delete table"
if [[ $REPLICATION_TRESHOLD != '0' ]]
then
titlecheck "MySQL / MariaDB replication"
SLAVE_STATUS=$($CONNECT "SHOW SLAVE STATUS\G" |grep -v row)
LAST_ERRNO=$(echo "$SLAVE_STATUS" |grep "Last_Errno:" |awk '{print $2}')
if [[ $LAST_ERRNO = 0 ]]
then
format N "Last_Errno:"
format O "OK"
else
format N "Last_Errno:"
format E "ERROR"
fi
echo '
'
REPLICATION_LATE=$(echo "$SLAVE_STATUS" |grep "Seconds_Behind_Master:" | awk '{ print $2 }' )
if [[ $REPLICATION_LATE == "NULL" ]]
then
format N "Second(s)_late:"
format E "ERROR"
elif [[ $REPLICATION_LATE -gt $REPLICATION_TRESHOLD ]]
then
format N "Second(s)_late:"
format E "ERROR ($REPLICATION_LATE)"
else
format N "Second(s)_late:"
format O "OK ($REPLICATION_LATE)"
fi
echo '
'
SLAVE_IO_RUNNING=$(echo "$SLAVE_STATUS" | grep "Slave_IO_Running:" | awk '{ print $2 }' )
if [[ $SLAVE_IO_RUNNING = "Yes" ]]
then
format N "S_IO_Running:"
format O "OK"
else
format N "S_IO_Running:"
format E "ERROR"
fi
echo '
'
SLAVE_SQL_RUNNING=$(echo "$SLAVE_STATUS" | grep "Slave_SQL_Running:" | awk '{ print $2 }')
if [[ $SLAVE_SQL_RUNNING = "Yes" ]]
then
format N "S_SQL_Running"
format O "OK"
else
format N "S_SQL_Running"
format E "ERROR"
fi
echo '
'
else
true
fi
}
function redis_check()
{
IP="$1"
PORT="$2"
titlecheck "Redis"
REDIS=$(echo ping |timeout 1 bash -c " /dev/null; echo "$?")
checktest "$REDIS" "Redis connect (port $PORT)"
}
function smtp_check()
{
IP="$1"
PORT="$2"
titlecheck "Postfix SMTP"
POSTFIX_SMTP=$(echo quit |timeout 1 bash -c " /dev/null; echo "$?")
checktest "$POSTFIX_SMTP" "Postfix SMTP connect (port 25)"
}
function vip_check()
{
titlecheck "VIP"
for VIP in "$@"
do
CHECK_VIP_PING=$(ping -c 1 -W 1 $VIP > /dev/null; echo "$?")
checktest "$CHECK_VIP_PING" "$VIP"
done
}
startpage "StackX Local monitoring"
title "StackX Monitoring System"
table_init
titlecheck "Server processes"
all_process_check
internet_check 1.1.1.1 8.8.8.8
vip_check 1.1.1.1
dns_check google.com cloudflare.com
fs_check
center_column
mysql_check $LOGINSXMYSQL $PASSSXMYSQL 0
memcached_check 127.0.0.1 11211
smtp_check 127.0.0.1 25
#elastic_check 127.0.0.1 9200
#redis_check 127.0.0.1 6379
disk_check 95 90 95 90
mem_check 95 90
load_check
check_update 10
table_end
endpage