#!/bin/bash
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
#
# StackX Monitoring System (SMS)
# sms.sh / sms.sh
# Author: Christophe Casalegno / Brain 0verride
# Contact: brain@christophe-casalegno.com
# Version 1.1.0
#
# Copyright (c) 2020 Christophe Casalegno
#
# This program is free software: you can redistribute it and/or modify
#
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see
#
# The license is available on this server here:
# https://www.christophe-casalegno.com/licences/gpl-3.0.txt
#
# To inlude in a php page for example: (dont forget to put your script in a
# directory not readable by your web server)
#
# The graphic banner is available here:
# https://www.christophe-casalegno.com/banners/ScalarX.jpg
#
#
#
# Example for a CONF_FILE: (with only needed variable for this script)
# loginsxmysql:sx
# passsxmysql:prxotpoofas
#
# Example for process2monitor.txt (generated automatically by my deployer for each server)
# apache2
# fail2ban-server
# memcached
# miniserv.pl
# php-fpm5.6
# php-fpm7.0
# php-fpm7.1
# php-fpm7.2
# php-fpm7.3
# php-fpm7.4
# php-fpm8.0
# pure-ftpd
# sshd
#
# Example for fs_list.txt (generated automatically by my deployer for each server)
#
# grep -v '#' /etc/fstab |awk '{print $2}' > fs_list.txt
#
# If you use other filesystem like network sshfs you may add:
# grep 'sshfs' /etc/fstab |awk '{print $2}' >> fs_list.txt
#
# Result can be something like :
# /
# /datastore
# /datadrop/cd101
# etc. depending of your configuration
#
# Need to check more tcp services on a limited system without netcat, etc.?
# You can use directly:
# timeout 1 cat $CHAIN2FORMAT"
elif [[ "$TARGETFORMAT" = 'O' ]]
then
echo "$CHAIN2FORMAT"
elif [[ "$TARGETFORMAT" = 'W' ]]
then
echo "$CHAIN2FORMAT"
elif [[ "$TARGETFORMAT" = 'E' ]]
then
echo "$CHAIN2FORMAT"
else
echo 'format not specified'
fi
}
function checktest()
{
if [ "$1" -eq 0 ]
then
format N "$2:"
format O "OK"
else
format N "$2:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("$2")
fi
echo ' '
}
function checkwarning()
{
if [ "$1" -eq 0 ]
then
format N "$2:"
format O "OK"
else
format N "$2:"
format W "WARNING"
fi
echo ' '
}
function startpage()
{
HTML_TITLE="$1"
echo ''
echo ''
echo "$HTML_TITLE"
echo ''
echo ''
}
function title()
{
TITLE="$1"
echo '
'
echo ''
echo '
'
echo "$TITLE"
echo '
'
echo ''
}
function titlecheck()
{
TITLE_CHECK="$1"
echo " "
format N "$TITLE_CHECK"
echo ""
}
function table_init()
{
echo '
'
}
function center_column()
{
echo '
'
echo '
'
echo '
'
}
function table_end()
{
echo '
'
}
function endpage()
{
echo "";
echo "";
}
function all_process_check()
{
for LINE2CHECK in $FILE2CHECK
do
check_process "$LINE2CHECK"
echo " "
done
}
function check_process()
{
P2CHECK="$1"
PROCESS=$(pgrep -c "$P2CHECK")
if [[ "$PROCESS" -eq 0 ]]
then
format N "$P2CHECK:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("$P2CHECK")
else
format N "$P2CHECK:"
format O "OK"
fi
}
function check_update()
{
DAYS_ALERT="$1"
LAST_UPDATE=""
echo ' '
titlecheck "System updates"
if [[ -z "$DAYS_ALERT" || ! "$DAYS_ALERT" =~ ^[0-9]+$ ]]
then
DAYS_ALERT=10
fi
if command -v zcat >/dev/null 2>&1
then
LAST_UPDATE=$(zcat -f /var/log/apt/history.log* 2>/dev/null |awk '/^Start-Date:/ {print $2}' |sort |tail -n 1)
else
LAST_UPDATE=$(grep -h "^Start-Date:" /var/log/apt/history.log /var/log/apt/history.log.[0-9]* 2>/dev/null |awk '{print $2}' |sort |tail -n 1)
fi
if [[ -z "$LAST_UPDATE" ]]
then
format N "Last apt updates:"
format W "WARNING (no record)"
return
fi
LAST_DATE=$(date -d "$LAST_UPDATE" +%s 2>/dev/null)
NOW_DATE=$(date +%s)
if [[ -z "$LAST_DATE" || "$LAST_DATE" -eq 0 ]]
then
format N "Last apt updates:"
format W "Unknown (date parse failed)"
return
fi
DIFF_DAYS=$(( (NOW_DATE - LAST_DATE) / 86400 ))
if [[ $DIFF_DAYS -gt $DAYS_ALERT ]]
then
format N "Last apt updates ($LAST_UPDATE, ${DIFF_DAYS} days ago):"
format W "WARNING"
else
format N "Last apt updates ($LAST_UPDATE, ${DIFF_DAYS} days ago):"
format O "OK"
fi
}
function storage_check()
{
SPACE_ERROR_TRESHOLD="$1"
SPACE_WARNING_TRESHOLD="$2"
INODES_ERROR_TRESHOLD="$3"
INODES_WARNING_TRESHOLD="$4"
titlecheck "Storages"
LIST_DISK=$(df -x tmpfs -x devtmpfs -x squashfs | grep 'dev' |awk -F " " '{print $1}' |cut -d/ -f3)
for DISK in $LIST_DISK
do
SPACE_USED_PERCENT=$(df |grep -w "$DISK" |head -1|awk -F" " '{print $5}' |cut -d% -f1)
SPACE_USED=$(df -x tmpfs -x devtmpfs -x squashfs -h |grep "$DISK" |head -1 |awk -F " " '{print $3}')
SPACE_TOTAL=$(df -x tmpfs -x devtmpfs -x squashfs -h |grep "$DISK" |head -1 |awk -F " " '{print $2}')
if [[ "$SPACE_USED_PERCENT" -gt "$SPACE_ERROR_TRESHOLD" ]]
then
format N "Part: $DISK - $SPACE_USED / $SPACE_TOTAL ($SPACE_USED_PERCENT%) space:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("$DISK")
echo ' '
elif [[ "$SPACE_USED_PERCENT" -gt "$SPACE_WARNING_TRESHOLD" ]]
then
format N "Part: $DISK - $SPACE_USED / $SPACE_TOTAL ($SPACE_USED_PERCENT%) space:"
format W "WARNING"
echo ' '
else
format N "Part: $DISK - $SPACE_USED / $SPACE_TOTAL ($SPACE_USED_PERCENT%) space:"
format O "OK"
echo ' '
fi
INODES_USED_PERCENT=$(df -i|grep -w "$DISK" |head -1 |awk -F" " '{print $5}' |cut -d% -f1)
INODES_USED=$(df -i -h |grep -w "$DISK" |head -1 |awk -F" " '{print $3}' |cut -d% -f1)
INODES_TOTAL=$(df -i -h |grep -w "$DISK" |head -1 |awk -F" " '{print $2}')
if [[ ! "$INODES_USED_PERCENT" =~ ^[0-9]+$ ]]
then
INODES_USED="-"
INODES_TOTAL="-"
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL (n/a) inodes:"
format O "N/A"
echo ' '
elif [[ "$INODES_USED_PERCENT" -gt "$INODES_ERROR_TRESHOLD" ]]
then
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL ($INODES_USED_PERCENT%) inodes:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("$DISK")
echo ' '
elif [[ "$INODES_USED_PERCENT" -gt "$INODES_WARNING_TRESHOLD" ]]
then
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL ($INODES_USED_PERCENT%) inodes:"
format W "WARNING"
echo ' '
else
format N "Part: $DISK - $INODES_USED / $INODES_TOTAL ($INODES_USED_PERCENT%) inodes:"
format O "OK"
echo ' '
fi
done
}
function dns_check()
{
titlecheck "DNS resolution"
for DNS in "$@"
do
CHECK_DNS_RESOLUTION=$(dig +time=0 +tries=1 $DNS > /dev/null; echo "$?")
checkwarning "$CHECK_DNS_RESOLUTION" "$DNS host"
done
}
function elastic_open_check()
{
IP="$1"
PORT="$2"
ELASTIC_URL="http://$IP:$PORT"
ELASTIC_HTTP_CODE=""
ELASTIC_HEALTH=""
ELASTIC_STATUS=""
ELASTIC_TIMEOUT=""
ELASTIC_NODES=""
titlecheck "(Elastic / Open) Search"
ELASTIC_HTTP_CODE=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 "$ELASTIC_URL/" 2>/dev/null || echo "000")
if [[ "$ELASTIC_HTTP_CODE" = "401" || "$ELASTIC_HTTP_CODE" = "403" ]]
then
format N "Elastic/Open Search endpoint (HTTP $ELASTIC_HTTP_CODE):"
format W "WARNING (auth required)"
echo ' '
return
fi
if [[ ! "$ELASTIC_HTTP_CODE" =~ ^[0-9]{3}$ || "$ELASTIC_HTTP_CODE" -lt 200 || "$ELASTIC_HTTP_CODE" -ge 400 ]]
then
format N "Elastic/Open Search endpoint (HTTP $ELASTIC_HTTP_CODE):"
format E "ERROR"
STATUS=1
ERROR_LIST+=("elastic")
echo ' '
return
fi
format N "Elastic Search endpoint (HTTP $ELASTIC_HTTP_CODE):"
format O "OK"
echo ' '
ELASTIC_HEALTH=$(curl -sS --max-time 3 "$ELASTIC_URL/_cluster/health?local=true&filter_path=status,timed_out,number_of_nodes" 2>/dev/null)
if [[ -z "$ELASTIC_HEALTH" ]]
then
format N "Elastic cluster health:"
format W "WARNING (no response)"
echo ' '
return
fi
ELASTIC_STATUS=$(echo "$ELASTIC_HEALTH" |grep -o '"status":"[^"]*"' |head -n 1 |cut -d '"' -f4)
ELASTIC_TIMEOUT=$(echo "$ELASTIC_HEALTH" |grep -o '"timed_out":[^,}]*' |head -n 1 |cut -d ':' -f2 |tr -d ' ')
ELASTIC_NODES=$(echo "$ELASTIC_HEALTH" |grep -o '"number_of_nodes":[0-9]*' |head -n 1 |cut -d ':' -f2)
if [[ "$ELASTIC_STATUS" = "red" ]]
then
format N "Elastic cluster health (status=$ELASTIC_STATUS, nodes=${ELASTIC_NODES:-?}):"
format E "ERROR"
STATUS=1
ERROR_LIST+=("elastic cluster")
elif [[ "$ELASTIC_STATUS" = "yellow" || "$ELASTIC_STATUS" = "green" ]]
then
if [[ "$ELASTIC_TIMEOUT" = "true" ]]
then
format N "Elastic cluster health (status=$ELASTIC_STATUS, nodes=${ELASTIC_NODES:-?}, timed_out=true):"
format W "WARNING"
else
format N "Elastic cluster health (status=$ELASTIC_STATUS, nodes=${ELASTIC_NODES:-?}):"
format O "OK"
fi
else
format N "Elastic cluster health:"
format W "WARNING (unknown response)"
fi
echo ' '
}
function opensearch_check()
{
IP="$1"
PORT="$2"
OPENSEARCH_SCHEME="https"
OPENSEARCH_HTTP_CODE=""
OPENSEARCH_HEALTH=""
OPENSEARCH_STATUS=""
OPENSEARCH_TIMEOUT=""
OPENSEARCH_CURL_AUTH=()
titlecheck "OpenSearch"
if [[ -z "${OPENSEARCH_PASSWORD:-}" && -n "${OPENSEARCH_ADMIN_PASSWORD:-}" ]]
then
OPENSEARCH_PASSWORD="$OPENSEARCH_ADMIN_PASSWORD"
fi
if [[ "${OPENSEARCH_SECURITY_ENABLED:-1}" = "1" ]]
then
if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]
then
format N "OpenSearch endpoint (port $PORT):"
format W "WARNING (missing password)"
echo ' '
return
fi
OPENSEARCH_CURL_AUTH=(-k -u "admin:$OPENSEARCH_PASSWORD")
else
OPENSEARCH_SCHEME="http"
fi
OPENSEARCH_HTTP_CODE=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "${OPENSEARCH_CURL_AUTH[@]}" "$OPENSEARCH_SCHEME://$IP:$PORT/" 2>/dev/null || echo "000")
if [[ ! "$OPENSEARCH_HTTP_CODE" =~ ^[0-9]{3}$ || "$OPENSEARCH_HTTP_CODE" -lt 200 || "$OPENSEARCH_HTTP_CODE" -ge 400 ]]
then
format N "OpenSearch endpoint (HTTP $OPENSEARCH_HTTP_CODE):"
format E "ERROR"
STATUS=1
ERROR_LIST+=("opensearch")
echo ' '
return
fi
format N "OpenSearch endpoint (HTTP $OPENSEARCH_HTTP_CODE):"
format O "OK"
echo ' '
OPENSEARCH_HEALTH=$(curl -sS --max-time 5 "${OPENSEARCH_CURL_AUTH[@]}" "$OPENSEARCH_SCHEME://$IP:$PORT/_cluster/health?local=true&filter_path=status,timed_out" 2>/dev/null)
if [[ -z "$OPENSEARCH_HEALTH" ]]
then
format N "OpenSearch cluster health:"
format W "WARNING (no response)"
echo ' '
return
fi
OPENSEARCH_STATUS=$(echo "$OPENSEARCH_HEALTH" |grep -o '"status":"[^"]*"' |head -n 1 |cut -d '"' -f4)
OPENSEARCH_TIMEOUT=$(echo "$OPENSEARCH_HEALTH" |grep -o '"timed_out":[^,}]*' |head -n 1 |cut -d ':' -f2 |tr -d ' ')
if [[ "$OPENSEARCH_STATUS" = "red" ]]
then
format N "OpenSearch cluster health (status=$OPENSEARCH_STATUS):"
format E "ERROR"
STATUS=1
ERROR_LIST+=("opensearch cluster")
elif [[ "$OPENSEARCH_STATUS" = "yellow" || "$OPENSEARCH_STATUS" = "green" ]]
then
if [[ "$OPENSEARCH_TIMEOUT" = "true" ]]
then
format N "OpenSearch cluster health (status=$OPENSEARCH_STATUS, timed_out=true):"
format W "WARNING"
else
format N "OpenSearch cluster health (status=$OPENSEARCH_STATUS):"
format O "OK"
fi
else
format N "OpenSearch cluster health:"
format W "WARNING (unknown response)"
fi
echo ' '
}
function fs_check()
{
FS_ACTIVE="$1"
if [[ "$FS_ACTIVE" = "yes" ]]
then
titlecheck "Filesystems"
FS_LIST_FILE="/home/sx/data/fs_list.txt"
DF_MOUNTS=$(df -x tmpfs -x devtmpfs -x squashfs -P 2>/dev/null |awk 'NR>1 {print $6}')
if [[ ! -r "$FS_LIST_FILE" ]]
then
format N "$FS_LIST_FILE:"
format W "WARNING (missing)"
echo ' '
return
fi
while IFS= read -r LINE || [[ -n "$LINE" ]]
do
LINE="${LINE%%#*}"
LINE="${LINE#"${LINE%%[![:space:]]*}"}"
LINE="${LINE%"${LINE##*[![:space:]]}"}"
[[ -z "$LINE" ]] && continue
FS_TARGET=$(echo "$LINE" |awk '{print $NF}')
[[ -z "$FS_TARGET" ]] && continue
if echo "$DF_MOUNTS" |grep -Fxq -- "$FS_TARGET"
then
format N "$FS_TARGET:"
format O "OK"
echo ' '
else
format N "$FS_TARGET:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("$FS_TARGET")
echo ' '
fi
done < "$FS_LIST_FILE"
else
true
fi
}
function load_check()
{
THREADS=$(grep processor /proc/cpuinfo |wc -l)
LOAD_TRESHOLD=$((THREADS * 4))
WAIT_B4_CHECK=1
echo " "
titlecheck "Load Average"
LOAD_AVERAGE1=$(awk '{print int($1)}' < /proc/loadavg)
sleep "$WAIT_B4_CHECK"
LOAD_AVERAGE2=$(awk '{print int($1)}' < /proc/loadavg)
if [[ "$LOAD_AVERAGE1" -ge "$LOAD_TRESHOLD" ]]
then
if [[ "$LOAD_AVERAGE2" -ge "$LOAD_AVERAGE1" ]]
then
format N "Load average ($LOAD_AVERAGE2 / $LOAD_TRESHOLD) / $THREADS core(s):"
format E "ERROR"
STATUS=1
ERROR_LIST+=("load")
else
format N "Load average ($LOAD_AVERAGE2 / $LOAD_TRESHOLD) / $THREADS core(s) but going down:"
format W "WARNING"
fi
else
format N "Load average ($LOAD_AVERAGE2 / $LOAD_TRESHOLD) / $THREADS core(s):"
format O "OK"
fi
}
function memcached_check()
{
IP="$1"
PORT="$2"
titlecheck "Memcached"
MEMCACHED=$(echo stats |timeout 1 bash -c " /dev/null; echo "$?")
checktest "$MEMCACHED" "Memcached connect (port 11211)"
}
function mem_check()
{
MEM_ERROR_TRESHOLD="$1"
MEM_WARNING_TRESHOLD="$2"
MEM_TOTAL=$(free -h |grep Mem |awk '{print $2}' |cut -d "i" -f1)
titlecheck "Memory"
MEM_USED_PERCENT=$(free |awk 'FNR == 2 {print 100-(($2-$3)/$2)*100}' |cut -d "." -f1)
MEM_USED=$(free -h |grep Mem |awk '{print $3}' |cut -d "i" -f1)
if [[ "$MEM_USED_PERCENT" -gt "$MEM_ERROR_TRESHOLD" ]]
then
format N "Memory usage: $MEM_USED / $MEM_TOTAL ($MEM_USED_PERCENT%):"
format E "ERROR"
STATUS=1
ERROR_LIST+=("memory")
elif [[ "$MEM_USED_PERCENT" -gt "$MEM_WARNING_TRESHOLD" ]]
then
format N "Memory usage: $MEM_USED / $MEM_TOTAL ($MEM_USED_PERCENT%):"
format W "WARNING"
else
format N "Memory usage: $MEM_USED / $MEM_TOTAL ($MEM_USED_PERCENT%):"
format O "OK"
fi
}
function mysql_check()
{
if [[ "${DB_TYPE:-}" = "postgresql" ]]
then
return
fi
LOGINSXSQL="$1"
DBPASSWORDSQL="$2"
REPLICATION_TRESHOLD="$3"
DBSXSQL="sx"
if command -v mariadb >/dev/null 2>&1
then
SQL_CLI="mariadb"
elif command -v mysql >/dev/null 2>&1
then
SQL_CLI="mysql"
else
SQL_CLI=""
fi
CONNECT="$SQL_CLI -u$LOGINSXSQL --database=$DBSXSQL -p$DBPASSWORDSQL -e "
titlecheck "MySQL / MariaDB"
if [[ -z "$SQL_CLI" ]]
then
CONNECTION_SQL=1
else
CONNECTION_SQL=$($CONNECT "SHOW VARIABLES LIKE '%version%';" > /dev/null; echo "$?")
fi
checktest "$CONNECTION_SQL" "Connexion SQL"
SHOW_TABLE=$($CONNECT "SHOW TABLES;" > /dev/null; echo "$?")
checktest "$SHOW_TABLE" "Show tables"
if [[ $REPLICATION_TRESHOLD != '0' ]]
then
titlecheck "MySQL / MariaDB replication"
SLAVE_STATUS=$($CONNECT "SHOW SLAVE STATUS\G" |grep -v row)
LAST_ERRNO=$(echo "$SLAVE_STATUS" |grep "Last_Errno:" |awk '{print $2}')
if [[ $LAST_ERRNO = 0 ]]
then
format N "Last_Errno:"
format O "OK"
else
format N "Last_Errno:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("MySQL replication: Last_Errno")
fi
echo ' '
REPLICATION_LATE=$(echo "$SLAVE_STATUS" |grep "Seconds_Behind_Master:" | awk '{ print $2 }' )
if [[ $REPLICATION_LATE == "NULL" ]]
then
format N "Second(s)_late:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("MySQL replication: Seconds_Behind_Master=NULL")
elif [[ $REPLICATION_LATE -gt $REPLICATION_TRESHOLD ]]
then
format N "Second(s)_late:"
format E "ERROR ($REPLICATION_LATE)"
STATUS=1
ERROR_LIST+=("MySQL replication: $REPLICATION_LATE sec late")
else
format N "Second(s)_late:"
format O "OK ($REPLICATION_LATE)"
fi
echo ' '
SLAVE_IO_RUNNING=$(echo "$SLAVE_STATUS" | grep "Slave_IO_Running:" | awk '{ print $2 }' )
if [[ $SLAVE_IO_RUNNING = "Yes" ]]
then
format N "S_IO_Running:"
format O "OK"
else
format N "S_IO_Running:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("MySQL replication: S_IO_Running")
fi
echo ' '
SLAVE_SQL_RUNNING=$(echo "$SLAVE_STATUS" | grep "Slave_SQL_Running:" | awk '{ print $2 }')
if [[ $SLAVE_SQL_RUNNING = "Yes" ]]
then
format N "S_SQL_Running"
format O "OK"
else
format N "S_SQL_Running"
format E "ERROR"
STATUS=1
ERROR_LIST+=("MySQL replication: S_SQL_Running")
fi
echo ' '
else
true
fi
}
function postgresql_check()
{
IP="$1"
PORT="$2"
PG_DB="${POSTGRESQL_DATABASE:-sx}"
PG_USER="${POSTGRESQL_USER:-sx}"
PG_PASS="${POSTGRESQL_PASSWORD:-}"
PG_READY=1
PG_LOGIN=1
PG_RECOVERY=""
titlecheck "PostgreSQL"
if command -v pg_isready >/dev/null 2>&1
then
pg_isready -h "$IP" -p "$PORT" >/dev/null 2>&1
PG_READY="$?"
fi
checktest "$PG_READY" "PostgreSQL readiness (port $PORT)"
if [[ -z "$PG_PASS" ]]
then
format N "PostgreSQL login:"
format W "WARNING (missing password)"
echo ' '
return
fi
if command -v psql >/dev/null 2>&1
then
PGPASSWORD="$PG_PASS" psql -h "$IP" -p "$PORT" -U "$PG_USER" -d "$PG_DB" -Atc 'SELECT 1;' >/dev/null 2>&1
PG_LOGIN="$?"
checktest "$PG_LOGIN" "PostgreSQL login ($PG_USER/$PG_DB)"
PG_RECOVERY=$(PGPASSWORD="$PG_PASS" psql -h "$IP" -p "$PORT" -U "$PG_USER" -d "$PG_DB" -Atc 'SELECT pg_is_in_recovery();' 2>/dev/null || true)
if [[ "$PG_RECOVERY" = "t" ]]
then
format N "PostgreSQL role:"
format O "standby"
echo ' '
elif [[ "$PG_RECOVERY" = "f" ]]
then
format N "PostgreSQL role:"
format O "primary"
echo ' '
fi
else
format N "PostgreSQL login:"
format W "WARNING (psql missing)"
echo ' '
fi
}
function postgresql_replication_check()
{
REPLICATION_TRESHOLD="$1"
PG_DB="${POSTGRESQL_DATABASE:-sx}"
PG_USER="${POSTGRESQL_USER:-sx}"
PG_PASS="${POSTGRESQL_PASSWORD:-}"
PG_PORT="${POSTGRESQL_PORT:-5432}"
PG_RECOVERY=""
PG_RECEIVER=""
PG_SENDERS=""
PG_LAG=""
titlecheck "PostgreSQL replication"
if [[ -z "$PG_PASS" || ! -x "$(command -v psql 2>/dev/null)" ]]
then
format N "PostgreSQL replication:"
format W "WARNING (missing password or psql)"
echo ' '
return
fi
PG_RECOVERY=$(PGPASSWORD="$PG_PASS" psql -h 127.0.0.1 -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" -Atc 'SELECT pg_is_in_recovery();' 2>/dev/null || true)
if [[ "$PG_RECOVERY" = "t" ]]
then
PG_RECEIVER=$(PGPASSWORD="$PG_PASS" psql -h 127.0.0.1 -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" -Atc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;" 2>/dev/null || true)
if [[ "$PG_RECEIVER" = "streaming" ]]
then
format N "WAL receiver:"
format O "streaming"
else
format N "WAL receiver:"
format E "ERROR (${PG_RECEIVER:-none})"
STATUS=1
ERROR_LIST+=("PostgreSQL replication receiver")
fi
echo ' '
return
fi
if [[ "$PG_RECOVERY" = "f" ]]
then
PG_SENDERS=$(PGPASSWORD="$PG_PASS" psql -h 127.0.0.1 -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" -Atc "SELECT count(*) FROM pg_stat_replication WHERE state='streaming';" 2>/dev/null || true)
if [[ "$PG_SENDERS" =~ ^[0-9]+$ && "$PG_SENDERS" -gt 0 ]]
then
format N "Streaming standby connections:"
format O "OK ($PG_SENDERS)"
else
format N "Streaming standby connections:"
format W "WARNING (${PG_SENDERS:-0})"
fi
echo ' '
PG_LAG=$(PGPASSWORD="$PG_PASS" psql -h 127.0.0.1 -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" -Atc "SELECT COALESCE(EXTRACT(EPOCH FROM max(replay_lag))::int,0) FROM pg_stat_replication;" 2>/dev/null || true)
if [[ "$PG_LAG" =~ ^[0-9]+$ && "$REPLICATION_TRESHOLD" != "0" && "$PG_LAG" -gt "$REPLICATION_TRESHOLD" ]]
then
format N "Replay lag:"
format E "ERROR (${PG_LAG}s)"
STATUS=1
ERROR_LIST+=("PostgreSQL replication lag")
elif [[ "$PG_LAG" =~ ^[0-9]+$ ]]
then
format N "Replay lag:"
format O "OK (${PG_LAG}s)"
fi
echo ' '
else
format N "PostgreSQL replication:"
format W "WARNING (role unknown)"
echo ' '
fi
}
function redis_check()
{
IP="$1"
PORT="$2"
REDIS_OUT=""
REDIS_RC=1
titlecheck "Redis"
if command -v redis-cli >/dev/null 2>&1
then
if [[ -n "${REDIS_PASSWORD:-}" ]]
then
REDIS_OUT=$(timeout 2 env REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h "$IP" -p "$PORT" ping 2>/dev/null || true)
else
REDIS_OUT=$(timeout 2 redis-cli -h "$IP" -p "$PORT" ping 2>/dev/null || true)
fi
if [[ "$REDIS_OUT" = "PONG" ]]
then
REDIS_RC=0
checktest "$REDIS_RC" "Redis PING/PONG (port $PORT)"
return
elif [[ "$REDIS_OUT" = NOAUTH* ]]
then
format N "Redis PING (port $PORT):"
format W "WARNING (auth required)"
echo ' '
return
fi
fi
REDIS_OUT=$(printf '*1\r\n$4\r\nPING\r\n' |timeout 2 bash -c "exec 3<>/dev/tcp/$IP/$PORT; cat >&3; timeout 1 cat <&3" 2>/dev/null || true)
if echo "$REDIS_OUT" |grep -q '^+PONG'
then
REDIS_RC=0
checktest "$REDIS_RC" "Redis PING/PONG (port $PORT)"
elif echo "$REDIS_OUT" |grep -q '^-NOAUTH'
then
format N "Redis PING (port $PORT):"
format W "WARNING (auth required)"
echo ' '
else
checktest 1 "Redis connect (port $PORT)"
fi
}
function smtp_check()
{
IP="$1"
PORT="$2"
titlecheck "Postfix SMTP"
POSTFIX_SMTP=$(echo quit |timeout 1 bash -c " /dev/null; echo "$?")
checktest "$POSTFIX_SMTP" "Postfix SMTP connect (port 25)"
}
function network_check()
{
TITLE="$1"
shift
titlecheck "$TITLE"
for VIP in "$@"
do
CHECK_VIP_PING=$(ping -c 1 -W 1 $VIP > /dev/null; echo "$?")
checktest "$CHECK_VIP_PING" "$VIP"
done
}
function raid_check()
{
mapfile -t RAID_LIST < <(awk '/^md[0-9]+[[:space:]]/ {print $1}' /proc/mdstat)
titlecheck "Raid"
for RAID in "${RAID_LIST[@]}"
do
CHECK_RAID="$(awk -v raid="$RAID" '
$1 == raid { in_block=1; next }
in_block && NF == 0 { exit }
in_block { print }
' /proc/mdstat)"
if [[ "$CHECK_RAID" == *"_"* ]]
then
format N "$RAID:"
format E "ERROR"
STATUS=1
ERROR_LIST+=("RAID degraded: $RAID")
else
format N "$RAID:"
format O "OK"
fi
echo " "
done
}
function zfs_check()
{
titlecheck "ZFS"
ZPOOL_BIN=""
ZPOOL_STATUS=""
if [[ -x "/usr/sbin/zpool" ]]
then
ZPOOL_BIN="/usr/sbin/zpool"
elif command -v zpool >/dev/null 2>&1
then
ZPOOL_BIN="$(command -v zpool)"
fi
format N "zpool status:"
if [[ -z "$ZPOOL_BIN" ]]
then
format E "ERROR"
STATUS=1
ERROR_LIST+=("ZFS zpool command not found")
echo " "
return
fi
ZPOOL_STATUS="$("$ZPOOL_BIN" status -x 2>/dev/null)"
ZPOOL_RC="$?"
if [[ "$ZPOOL_RC" -ne 0 ]]
then
format E "ERROR"
STATUS=1
ERROR_LIST+=("ZFS status check failed")
elif [[ "$ZPOOL_STATUS" = "all pools are healthy" ]]
then
format O "OK"
else
format E "ERROR"
STATUS=1
ERROR_LIST+=("ZFS error")
fi
echo " "
}
function backup_cfg_value()
{
BACKUP_CFG_FILE="$1"
BACKUP_CFG_KEY="$2"
BACKUP_CFG_DEFAULT="$3"
BACKUP_CFG_VALUE=""
if [[ -r "$BACKUP_CFG_FILE" ]]
then
BACKUP_CFG_VALUE=$(grep -E "^[[:space:]]*$BACKUP_CFG_KEY:" "$BACKUP_CFG_FILE" 2>/dev/null |tail -n 1 |cut -d ":" -f2-)
BACKUP_CFG_VALUE="${BACKUP_CFG_VALUE#"${BACKUP_CFG_VALUE%%[![:space:]]*}"}"
BACKUP_CFG_VALUE="${BACKUP_CFG_VALUE%"${BACKUP_CFG_VALUE##*[![:space:]]}"}"
fi
if [[ -z "$BACKUP_CFG_VALUE" ]]
then
BACKUP_CFG_VALUE="$BACKUP_CFG_DEFAULT"
fi
echo "$BACKUP_CFG_VALUE"
}
function backup_file_age_hours()
{
BACKUP_FILE="$1"
BACKUP_NOW_EPOCH=$(date +%s)
BACKUP_FILE_EPOCH=$(stat -c %Y "$BACKUP_FILE" 2>/dev/null)
if [[ -z "$BACKUP_FILE_EPOCH" ]]
then
echo "-1"
return
fi
echo $(( (BACKUP_NOW_EPOCH - BACKUP_FILE_EPOCH) / 3600 ))
}
function backup_report()
{
BACKUP_LABEL="$1"
BACKUP_STATE="$2"
BACKUP_DETAILS="$3"
format N "$BACKUP_LABEL$BACKUP_DETAILS:"
if [[ "$BACKUP_STATE" = "OK" ]]
then
format O "OK"
elif [[ "$BACKUP_STATE" = "WARNING" ]]
then
format W "WARNING"
else
format E "ERROR"
STATUS=1
ERROR_LIST+=("$BACKUP_LABEL")
fi
echo ' '
}
function backup_uptime_hours()
{
awk '{print int($1/3600)}' /proc/uptime 2>/dev/null
}
function backup_missing_report()
{
BACKUP_LABEL="$1"
BACKUP_ERROR_HOURS="$2"
BACKUP_REASON="$3"
BACKUP_UPTIME_HOURS=$(backup_uptime_hours)
if [[ "$BACKUP_UPTIME_HOURS" =~ ^[0-9]+$ ]] && [[ "$BACKUP_UPTIME_HOURS" -lt "$BACKUP_ERROR_HOURS" ]]
then
backup_report "$BACKUP_LABEL" "WARNING" " ($BACKUP_REASON, uptime ${BACKUP_UPTIME_HOURS}h)"
else
backup_report "$BACKUP_LABEL" "ERROR" " ($BACKUP_REASON)"
fi
}
function backup_script_running()
{
BACKUP_SCRIPT_NAME="$1"
BACKUP_LOCK_FILE="/run/lock/${BACKUP_SCRIPT_NAME}.lock"
BACKUP_SCRIPT_PID=""
if [[ -r "$BACKUP_LOCK_FILE" ]]
then
BACKUP_SCRIPT_PID=$(head -n 1 "$BACKUP_LOCK_FILE" 2>/dev/null |tr -d '[:space:]')
if [[ "$BACKUP_SCRIPT_PID" =~ ^[0-9]+$ ]] && kill -0 "$BACKUP_SCRIPT_PID" 2>/dev/null
then
return 0
fi
fi
if command -v pgrep >/dev/null 2>&1
then
if pgrep -f "/usr/local/bin/$BACKUP_SCRIPT_NAME" >/dev/null 2>&1
then
return 0
fi
fi
return 1
}
function backup_log_freshness_check()
{
BACKUP_LABEL="$1"
BACKUP_GLOB="$2"
BACKUP_WARNING_HOURS="$3"
BACKUP_ERROR_HOURS="$4"
BACKUP_LAST_LOG=""
BACKUP_LATEST_LOG=$(ls -1t $BACKUP_GLOB 2>/dev/null |head -n 1)
if [[ -z "$BACKUP_LATEST_LOG" ]]
then
backup_missing_report "$BACKUP_LABEL" "$BACKUP_ERROR_HOURS" "missing"
return 1
fi
BACKUP_LOG_AGE_HOURS=$(backup_file_age_hours "$BACKUP_LATEST_LOG")
BACKUP_LOG_BASENAME=$(basename "$BACKUP_LATEST_LOG")
if [[ "$BACKUP_LOG_AGE_HOURS" -gt "$BACKUP_ERROR_HOURS" ]]
then
backup_report "$BACKUP_LABEL" "ERROR" " ($BACKUP_LOG_BASENAME, ${BACKUP_LOG_AGE_HOURS}h)"
elif [[ "$BACKUP_LOG_AGE_HOURS" -gt "$BACKUP_WARNING_HOURS" ]]
then
backup_report "$BACKUP_LABEL" "WARNING" " ($BACKUP_LOG_BASENAME, ${BACKUP_LOG_AGE_HOURS}h)"
else
backup_report "$BACKUP_LABEL" "OK" " ($BACKUP_LOG_BASENAME, ${BACKUP_LOG_AGE_HOURS}h)"
fi
BACKUP_LAST_LOG="$BACKUP_LATEST_LOG"
return 0
}
function db_backup_check()
{
BACKUP_WARNING_HOURS="$1"
BACKUP_ERROR_HOURS="$2"
DB_BACKUP_SCRIPT="sxbkpmdb"
DB_BACKUP_CFG="/etc/sx/sxbkpmdb/sxbkpmdb.cfg"
DB_BACKUP_GLOB="/var/log/sxbkpmdb/sxbkpmdb-*.log"
DB_BACKUP_PATTERN="*.sql.gz"
DB_LOG=""
if [[ "${DB_TYPE:-}" = "postgresql" ]]
then
DB_BACKUP_SCRIPT="sxbkppgdb"
DB_BACKUP_CFG="/etc/sx/sxbkppgdb/sxbkppgdb.cfg"
DB_BACKUP_GLOB="/var/log/sxbkppgdb/sxbkppgdb-*.log"
DB_BACKUP_PATTERN="*.dump"
fi
if backup_log_freshness_check "DB log" "$DB_BACKUP_GLOB" "$BACKUP_WARNING_HOURS" "$BACKUP_ERROR_HOURS"
then
DB_LOG="$BACKUP_LAST_LOG"
fi
if [[ -n "$DB_LOG" ]]
then
DB_LOG_AGE_HOURS=$(backup_file_age_hours "$DB_LOG")
if grep -q "\[INF\] Done" "$DB_LOG" 2>/dev/null
then
backup_report "DB backup completion" "OK" ""
elif backup_script_running "$DB_BACKUP_SCRIPT"
then
backup_report "DB backup completion" "WARNING" " (running)"
elif [[ "$DB_LOG_AGE_HOURS" =~ ^-?[0-9]+$ ]] && [[ "$DB_LOG_AGE_HOURS" -ge 0 ]] && [[ "$DB_LOG_AGE_HOURS" -le 1 ]]
then
backup_report "DB backup completion" "WARNING" " (pending)"
else
backup_report "DB backup completion" "ERROR" " (Done marker missing)"
fi
fi
DB_BACKUP_DIR=$(backup_cfg_value "$DB_BACKUP_CFG" "BKP_DIR" "/home/sxadmin/sxbackup")
DB_LATEST_DUMP=$(find "$DB_BACKUP_DIR" -mindepth 2 -maxdepth 2 -type f -name "$DB_BACKUP_PATTERN" -printf '%T@ %p\n' 2>/dev/null |sort -nr |head -n 1 |cut -d " " -f2-)
if [[ -z "$DB_LATEST_DUMP" ]]
then
backup_missing_report "DB dump" "$BACKUP_ERROR_HOURS" "missing"
return
fi
DB_DUMP_AGE_HOURS=$(backup_file_age_hours "$DB_LATEST_DUMP")
DB_DUMP_BASENAME=$(basename "$DB_LATEST_DUMP")
if [[ "$DB_DUMP_AGE_HOURS" -gt "$BACKUP_ERROR_HOURS" ]]
then
backup_report "DB dump" "ERROR" " ($DB_DUMP_BASENAME, ${DB_DUMP_AGE_HOURS}h)"
elif [[ "$DB_DUMP_AGE_HOURS" -gt "$BACKUP_WARNING_HOURS" ]]
then
backup_report "DB dump" "WARNING" " ($DB_DUMP_BASENAME, ${DB_DUMP_AGE_HOURS}h)"
else
backup_report "DB dump" "OK" " ($DB_DUMP_BASENAME, ${DB_DUMP_AGE_HOURS}h)"
fi
}
function rdiff_backup_check()
{
BACKUP_WARNING_HOURS="$1"
BACKUP_ERROR_HOURS="$2"
RDIFF_CFG='/etc/sx/sxfilesbkp/sxfilesbkp.cfg'
RDIFF_REMOTE_HOST=$(backup_cfg_value "$RDIFF_CFG" "REMOTE_HOST" "")
RDIFF_REMOTE_USER=$(backup_cfg_value "$RDIFF_CFG" "REMOTE_USER" "")
if [[ -z "$RDIFF_REMOTE_HOST" || -z "$RDIFF_REMOTE_USER" || "$RDIFF_REMOTE_HOST" = "backup_server" || "$RDIFF_REMOTE_USER" = "server_username" ]]
then
backup_report "Rdiff backup configuration" "WARNING" " (not configured)"
return
fi
RDIFF_LOG=""
if backup_log_freshness_check "Rdiff log" "/var/log/sxfilesbkp/sxfilesbkp-*.log" "$BACKUP_WARNING_HOURS" "$BACKUP_ERROR_HOURS"
then
RDIFF_LOG="$BACKUP_LAST_LOG"
fi
if [[ -z "$RDIFF_LOG" ]]
then
return
fi
RDIFF_LOG_AGE_HOURS=$(backup_file_age_hours "$RDIFF_LOG")
if grep -q "\[INF\] Done" "$RDIFF_LOG" 2>/dev/null
then
backup_report "Rdiff backup completion" "OK" ""
elif backup_script_running "sxfilesbkp"
then
backup_report "Rdiff backup completion" "WARNING" " (running)"
elif [[ "$RDIFF_LOG_AGE_HOURS" =~ ^-?[0-9]+$ ]] && [[ "$RDIFF_LOG_AGE_HOURS" -ge 0 ]] && [[ "$RDIFF_LOG_AGE_HOURS" -le 1 ]]
then
backup_report "Rdiff backup completion" "WARNING" " (pending)"
else
backup_report "Rdiff backup completion" "ERROR" " (Done marker missing)"
fi
if grep -q "\[INF\] Backup: " "$RDIFF_LOG" 2>/dev/null
then
backup_report "Rdiff backup execution" "OK" ""
elif backup_script_running "sxfilesbkp"
then
backup_report "Rdiff backup execution" "WARNING" " (running)"
elif [[ "$RDIFF_LOG_AGE_HOURS" =~ ^-?[0-9]+$ ]] && [[ "$RDIFF_LOG_AGE_HOURS" -ge 0 ]] && [[ "$RDIFF_LOG_AGE_HOURS" -le 1 ]]
then
backup_report "Rdiff backup execution" "WARNING" " (pending)"
else
backup_report "Rdiff backup execution" "ERROR" " (no backup run marker)"
fi
}
function backup_check()
{
BACKUP_WARNING_HOURS="$1"
BACKUP_ERROR_HOURS="$2"
titlecheck "Backups"
DB_BACKUP_BIN="/usr/local/bin/sxbkpmdb"
if [[ "${DB_TYPE:-}" = "postgresql" ]]
then
DB_BACKUP_BIN="/usr/local/bin/sxbkppgdb"
fi
if [[ -x "$DB_BACKUP_BIN" ]]
then
db_backup_check "$BACKUP_WARNING_HOURS" "$BACKUP_ERROR_HOURS"
else
backup_report "DB backup module" "WARNING" " ($(basename "$DB_BACKUP_BIN") not installed)"
fi
if [[ -x /usr/local/bin/sxfilesbkp ]]
then
rdiff_backup_check "$BACKUP_WARNING_HOURS" "$BACKUP_ERROR_HOURS"
else
backup_report "Rdiff backup module" "WARNING" " (sxfilesbkp not installed)"
fi
}
function log()
{
LOGDATE=$(date +"%y-%m-%d")
LOGDIR="/home/sx/log/sms"
mkdir -p "$LOGDIR" 2>/dev/null || true
if [[ ! -w "$LOGDIR" ]]
then
LOGDIR="/tmp/sx-sms"
mkdir -p "$LOGDIR" 2>/dev/null || true
fi
LOGFILE="$LOGDIR/$LOGDATE.log"
STATUS_TXT="OK"
if [[ $STATUS -ne 0 ]]
then
STATUS_TXT="ERROR"
errors_csv=$(IFS=, ; echo "${ERROR_LIST[*]}")
echo "$(date) - Check run - Status: $STATUS_TXT - Issues: $errors_csv" >> "$LOGFILE"
else
echo "$(date) - Check run - Status: $STATUS_TXT" >> "$LOGFILE"
fi
}
startpage "StackX Local monitoring"
title "StackX Monitoring System"
table_init
titlecheck "Server processes"
all_process_check
network_check "Internet" 1.1.1.1 8.8.8.8
network_check "localhost network" 127.0.0.1
#network_check VIP nosetup
dns_check google.com cloudflare.com
fs_check yes
storage_check 95 90 95 90
mem_check 95 90
load_check
center_column
mysql_check $LOGINSXMYSQL $PASSSXMYSQL 0
memcached_check 127.0.0.1 11211
smtp_check 127.0.0.1 25
backup_check 30 54
#elastic_open_check 127.0.0.1 9200
#opensearch_check 127.0.0.1 9200
#redis_check 127.0.0.1 6379
#postgresql_check 127.0.0.1 5432
#postgresql_replication_check 9600
#raid_check
#zfs_check
check_update 10
table_end
endpage
log
'
echo '