22

✏️ Editing: check_salt_running.sh

#!/usr/bin/env bash
set -u

SERVICE_NAME="salt-minion"
STATE_FILE="/opt/.saltrunning"
RESTART_STAMP="/run/check_salt_running.restart"
RECOVERY_GRACE=2700 # 45 minutes
RESTART_AGE=3600 # 60 minutes
WARN_AGE=5400 # 90 minutes
CRIT_AGE=7200 # 120 minutes

if (( $# > 0 )); then
  echo "UNKNOWN - this check does not accept arguments"
  exit 3
fi

RECOVERY_GRACE_MINUTES=$(( ${RECOVERY_GRACE} / 60 ))
RESTART_AGE_MINUTES=$(( ${RESTART_AGE} / 60 ))
WARN_AGE_MINUTES=$(( ${WARN_AGE} / 60 ))
CRIT_AGE_MINUTES=$(( ${CRIT_AGE} / 60 ))

if ! command -v systemctl >/dev/null 2>&1; then
  echo "UNKNOWN - systemctl not available"
  exit 3
fi

get_service_uptime_seconds() {
  local active_us now_us

active_us="$(systemctl show -p ActiveEnterTimestampMonotonic --value "${SERVICE_NAME}" 2>/dev/null || true)"
  if [[ ! "${active_us}" =~ ^[0-9]+$ || "${active_us}" -le 0 ]]; then
    return 1
  fi

now_us="$(awk '{printf "%.0f", $1 * 1000000}' /proc/uptime 2>/dev/null || true)"
  if [[ ! "${now_us}" =~ ^[0-9]+$ || "${now_us}" -lt "${active_us}" ]]; then
    return 1
  fi

echo $(( (${now_us} - ${active_us}) / 1000000 ))
}

get_host_uptime_seconds() {
  awk '{printf "%.0f", $1}' /proc/uptime 2>/dev/null
}

restart_if_service_uptime_exceeds_restart_age() {
  local service_uptime service_uptime_minutes restart_output stamp_age stamp_age_minutes

if service_uptime="$(get_service_uptime_seconds)"; then
    service_uptime_minutes=$(( ${service_uptime} / 60 ))
    if (( ${service_uptime} >= ${RESTART_AGE} )); then
      if [[ -e "${RESTART_STAMP}" ]]; then
        stamp_age=$(( $(date +%s) - $(stat -c '%Y' "${RESTART_STAMP}" 2>/dev/null || echo 0) ))
        stamp_age_minutes=$(( ${stamp_age} / 60 ))
        if (( ${stamp_age} >= 0 && ${stamp_age} < ${RESTART_AGE} )); then
          echo "; restart suppressed, last restart requested ${stamp_age_minutes} minutes ago"
          return
        fi
      fi

if restart_output="$(systemctl --no-block restart "${SERVICE_NAME}" 2>&1)"; then
        touch "${RESTART_STAMP}" 2>/dev/null || true
        echo "; restart initiated after ${service_uptime_minutes} minutes service uptime"
      else
        echo "; restart failed after ${service_uptime_minutes} minutes service uptime: ${restart_output}"
      fi
    else
      echo "; restart suppressed, service uptime ${service_uptime_minutes} minutes below threshold ${RESTART_AGE_MINUTES} minutes"
    fi
  else
    echo "; restart skipped, unable to determine service uptime"
  fi
}

service_load_state="$(systemctl show -p LoadState --value "${SERVICE_NAME}" 2>/dev/null || true)"
service_load_state="${service_load_state#LoadState=}"
if [[ -z "${service_load_state}" || "${service_load_state}" == "not-found" ]]; then
  echo "WARNING - ${SERVICE_NAME} is not installed or the unit is unavailable"
  exit 1
fi

service_state="$(systemctl is-active "${SERVICE_NAME}" 2>/dev/null || true)"
if [[ "${service_state}" != "active" ]]; then
  if [[ "${service_state}" == "activating" || "${service_state}" == "deactivating" || "${service_state}" == "reloading" ]]; then
    echo "WARNING - ${SERVICE_NAME} is in transient state: ${service_state:-unknown}"
    exit 1
  fi

echo "CRITICAL - ${SERVICE_NAME} is not active (state: ${service_state:-unknown})"
  exit 2
fi

if [[ ! -e "${STATE_FILE}" ]]; then
  echo "CRITICAL - ${STATE_FILE} does not exist"
  exit 2
fi

if ! last_run="$(stat -c '%Y' "${STATE_FILE}" 2>/dev/null)"; then
  echo "UNKNOWN - unable to read mtime for ${STATE_FILE}"
  exit 3
fi

now="$(date +%s)"
age=$(( ${now} - ${last_run} ))
age_minutes=$(( ${age} / 60 ))

if (( ${age} < 0 )); then
  echo "UNKNOWN - ${STATE_FILE} has a future timestamp"
  exit 3
fi

if host_uptime="$(get_host_uptime_seconds)" && [[ "${host_uptime}" =~ ^[0-9]+$ ]] && (( ${host_uptime} < ${RECOVERY_GRACE} )); then
  host_uptime_minutes=$(( ${host_uptime} / 60 ))
  echo "OK - host up for ${host_uptime_minutes} minutes, allowing ${RECOVERY_GRACE_MINUTES} minutes for Salt to update after recovery | saltrunning_age=${age_minutes}m;${WARN_AGE_MINUTES};${CRIT_AGE_MINUTES};0;"
  exit 0
fi

if (( ${age} >= ${CRIT_AGE} )); then
  restart_note="$(restart_if_service_uptime_exceeds_restart_age)"
  echo "CRITICAL - Salt has not updated for ${age_minutes} minutes (threshold ${CRIT_AGE_MINUTES} minutes)${restart_note} | saltrunning_age=${age_minutes}m;${WARN_AGE_MINUTES};${CRIT_AGE_MINUTES};0;"
  exit 2
fi

if (( ${age} >= ${WARN_AGE} )); then
  restart_note="$(restart_if_service_uptime_exceeds_restart_age)"
  echo "WARNING - Salt has not updated for ${age_minutes} minutes (threshold ${WARN_AGE_MINUTES} minutes)${restart_note} | saltrunning_age=${age_minutes}m;${WARN_AGE_MINUTES};${CRIT_AGE_MINUTES};0;"
  exit 1
fi

if (( ${age} >= ${RESTART_AGE} )); then
  restart_note="$(restart_if_service_uptime_exceeds_restart_age)"
  echo "OK - ${SERVICE_NAME} active, last update ${age_minutes} minutes ago${restart_note} | saltrunning_age=${age_minutes}m;${WARN_AGE_MINUTES};${CRIT_AGE_MINUTES};0;"
  exit 0
fi

echo "OK - ${SERVICE_NAME} active, last update ${age_minutes} minutes ago | saltrunning_age=${age_minutes}m;${WARN_AGE_MINUTES};${CRIT_AGE_MINUTES};0;"
exit 0

MR SKB