srvmon

#!/bin/bash
# Copyright 2020 - Remy van Elst - https://raymii.org/s/software/Bash_HTTP_Monitoring_Dashboard.html
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Afferro General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Start of configuration.

#array names must not contain spaces, only a-ZA-Z.
declare -A urls # do not remove this line
urls[dilbert.com]="https://dilbert.com"
urls[gist.github.com]="https://gist.github.com"
urls[github.com]="https://github.com"
urls[lobste.rs]="https://lobste.rs"
urls[raymii.org]="https://raymii.org"
urls[treesforlife.org.uk]="https://treesforlife.org.uk"
urls[tweakers.net]="https://tweakers.net"
urls[www.buienradar.nl]="https://www.buienradar.nl"
urls[www.google.com]="https://www.google.com"
urls[www.smbc-comics.com]="https://www.smbc-comics.com"
urls[xkcd.com]="https://xkcd.com"

# The default status code. Can be overridden per URL lower in the script.
defaultExpectedStatusCode=200

# Expected code for url, key must match urls[]. Only for URL's you consider UP, but for example require authentication
declare -A statuscode # do not remove this line
statuscode[gist.github.com]=302

# How many curl checks to run at the same time
maxConcurrentCurls=12

# Max timeout of a check in seconds
defaultTimeOut=10

# After how many seconds should we re-check any failed checks? (To prevent flapping)
flapRetry=5

# Title of the webpage
title="Status Dashboard"

# Use CGI mode
cgi=false

# Callback URL for failed checks. Script will do a POST request with
# the following parameters: urlencode(url), name, expected_status, actual_status, error.
callbackURL=""

# Start of script. Do not edit below
# Exit immediately if a command exits with a non-zero status.
set -e
# patterns which match no files expand to a null string, otherwise later on we'd have *.status as a file...
shopt -s nullglob

# helper functions
command_exists() {
    # check if command exists and fail otherwise
    command -v "$1" >/dev/null 2>&1
    if [[ $? -ne 0 ]]; then
        echo "I require '$1' but it's not installed. Please install it. Aborting."
        exit 1
    fi
}

# environment checks
[ "${BASH_VERSINFO:-0}" -lt 4 ] && { echo "I require at least Bash version 4. Aborting." >&2 ; exit 2; }
command_exists curl

case "$(uname -s)" in
  Darwin*)
    datecmd="gdate"
    command_exists "${datecmd}"
    ;;
  *)
    datecmd="date"
esac

# start of the function definitions

# This function allows the script to execute all the curl calls in parallel. 
# Otherwise, if one would timeout or take long, the rest after that would be
# slower. We're writing the status code to a file, reading that later on. Why? 
# Because an array cannot be filled via a subprocess (curl ... &)
doRequest() {
  name="${1}"
  url="${2}"
  checkStartTimeMs="$(${datecmd} +%s%3N)" # epoch in microseconds, but last chars stripped so it's milliseconds
  set +e # curl errors don't count for early exit, turn off e
  checkStatusCode="$(curl --max-time "${defaultTimeOut}" --silent --show-error --insecure --output /dev/null --write-out "%{http_code}" "$url" --stderr "${tempfolder}/FAIL/${name}.error")"
  set -e # turn e back on
  checkEndTimeMs="$(${datecmd} +%s%3N)"
  timeCheckTook="$((checkEndTimeMs-checkStartTimeMs))"

  expectedStatusCode=${defaultExpectedStatusCode}
  # check if there is a specific status code for this domain
  if [[ -n "${statuscode[${name}]}" ]]; then
    expectedStatusCode="${statuscode[${name}]}"
  fi 

  if [[ ${expectedStatusCode} -eq ${checkStatusCode} ]]; then
    echo "${timeCheckTook}" > "${tempfolder}/OK/${name}.duration"
    # remove any previous failed check attempt
    if [[ -r "${tempfolder}/FAIL/${name}.status" ]]; then
      rm "${tempfolder}/FAIL/${name}.status"
    fi
  else
    echo "${checkStatusCode}" > "${tempfolder}/FAIL/${name}.status"
    # remove any previous okay check
    if [[ -r "${tempfolder}/OK/${name}.duration" ]]; then
      rm "${tempfolder}/OK/${name}.duration";
    fi
  fi

  # if the error file is empty, remove it.
  if [[ ! -s "${tempfolder}/FAIL/${name}.error" ]]; then
    rm "${tempfolder}/FAIL/${name}.error" 
  fi
}

recheckFailedChecks() {
  pushd "${tempfolder}/FAIL" >/dev/null 2>&1
  failFiles=(*.status)   
  failCount=${#failFiles[@]}
  if [[ failCount -gt 0 ]]; then
    echo -e "\e[1;31mFailed checks, re-checking after ${flapRetry} seconds.\e[0m" >&2 # output to stderr
    sleep "${flapRetry}";
    for filename in "${failFiles[@]}"; do
      if [[ -r $filename ]]; then
        filenameWithoutExt=${filename%.*}
        if [[ "$(jobs | wc -l)" -ge ${maxConcurrentCurls} ]] ; then # run 12 curl commands at max parallel
            wait -n
        fi
        doRequest "${filenameWithoutExt}" "${urls[${filenameWithoutExt}]}"
      fi
    done
    wait
  fi
  popd >/dev/null 2>&1
}


writeOkayChecks() {
  echo "</div></div>"
  echo "<div class=row><div class=col>"
  pushd "${tempfolder}/OK" >/dev/null 2>&1
  okFiles=(*.duration)
  okCount=${#okFiles[@]}
  if [[ okCount -gt 0 ]]; then
    for filename in "${okFiles[@]}"; do
      if [[ -r $filename ]]; then
        value="$(cat "$filename")"
        filenameWithoutExt=${filename%.*}
        echo '<a href="#" class="btn btn-success disabled" tabindex="-1" role="button" aria-disabled="true" style="margin-top: 10px; padding: 10px;">'
        echo "${filenameWithoutExt}"
        echo "<font color=LightGray> ("
        echo "${value}"
        echo " ms)</font>"
        echo "</a> &nbsp;"
      fi
    done
  fi
  popd >/dev/null 2>&1
}

urlencode() {
  local LANG=C 
  local i='' 
  local c='' 
  local e=''
  for ((i=0; i<${#1}; i++)); do
    c=${1:$i:1}
    [[ "$c" =~ [a-zA-Z0-9\.\~\_\-] ]] || printf -v c '%%%02X' "'$c"
    e+="$c"
  done
  echo "$e"
}

doCallbackForFailedChecks() {
  # If the callback URL is empty, don't do anything.
  if [[ -z "${callbackURL}" ]]; then
    return;
  fi

  pushd "${tempfolder}/FAIL" >/dev/null 2>&1
  failFiles=(*.status)
  failCount=${#failFiles[@]}
  for filename in "${failFiles[@]}"; do
    if [[ -r $filename ]]; then
      filenameWithoutExt=${filename%.*}
      if [[ -r "${filenameWithoutExt}.error" ]]; then
        curlError="$(cat "${filenameWithoutExt}.error" 2>/dev/null)"
      else
        curlError="Status code does not match expected code"
      fi
      url="${urls[${filenameWithoutExt}]}"
      encodedUrl="$(urlencode ${url})"
      currentStatus="$(cat "${filename}")"
      expectedStatus="000"
      if [[ -z "${statuscode[${filenameWithoutExt}]}" ]]; then
        expectedStatus="${defaultExpectedStatusCode}"
      else
        expectedStatus="${statuscode[${filenameWithoutExt}]}"
      fi

      # If the callback URL is a slack webhook then structure payload as slack attachment message block instead
      slackRegex="^https\:\/\/hooks\.slack\.com"
      if [[ ${callbackURL} =~ ${slackRegex} ]]; then
        curlData="{\"attachments\":[{\"color\":\"e00000\",\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"*Resource Unavailable*\\n*url:* ${url}\\n*name:* ${filenameWithoutExt}\\n*expected_status:* ${expectedStatus}\\n*actual_status:* ${currentStatus}\\n*error:* ${curlError}\"}}]}]}"
      else
        curlData="{\"url\":\"${encodedUrl}\",\"name\":\"${filenameWithoutExt}\",\"expected_status\":\"${expectedStatus}\",\"actual_status\":\"${currentStatus}\",\"error\":\"${curlError}\"}";
      fi

      curl --max-time "${defaultTimeOut}" --silent --show-error --insecure --output /dev/null --request POST \
          -H "Content-Type: application/json" \
          --data "${curlData}" \
          "${callbackURL}"
    fi
  done
  popd >/dev/null 2>&1
}

writeFailedChecks() {
  pushd "${tempfolder}/FAIL" >/dev/null 2>&1
  failFiles=(*.status)
  failCount=${#failFiles[@]}
  if [[ failCount -gt 0 ]]; then
    echo '<div class="alert alert-danger" role="alert">'
    echo "Errors occured! ${failCount} check(s) have failed."
    echo "</div>"
    echo '<table class="table">'
    echo '<thead><tr>'
    echo '<th>Name</th>'
    echo '<th>HTTP Status/Expected</th>'
    echo '<th>Error</th>'
    echo '</tr></thead><tbody>'
    for filename in "${failFiles[@]}"; do
      if [[ -r $filename ]]; then
        filenameWithoutExt=${filename%.*}
        if [[ -r "${filenameWithoutExt}.error" ]]; then
          curlError="$(cat "${filenameWithoutExt}.error" 2>/dev/null)"
        else
          curlError="Status code does not match expected code"
        fi
        status="$(cat "${filename}")"
        echo "<tr class='table-danger'><td>"
        echo "${filenameWithoutExt}"
        echo "</td><td>"
        echo "${status} / "
        if [[ -z "${statuscode[${filenameWithoutExt}]}" ]]; then
          echo "${defaultExpectedStatusCode}"
        else
          echo "${statuscode[${filenameWithoutExt}]}"
        fi
        echo "</td><td>"
        echo "${curlError}"
        echo "</td></tr>"
      fi
    done
    echo "<tr><td colspan=3 class='small'>All failed checks were attempted again after ${flapRetry} seconds and failed again.</td></tr>"
    echo "</tbody></table>"
  else 
    echo '<div class="alert alert-success" role="alert">'
    echo "All is well, all services are up."
    echo "</div>"
  fi
  popd >/dev/null 2>&1
}

cleanupFailedCheckFiles() {
  pushd "${tempfolder}/FAIL" >/dev/null 2>&1
  errorFiles=(*.error)
  for filename in "${errorFiles[@]}"; do
    if [[ -r $filename ]]; then
      rm "${filename}"
    fi
  done

  statusFiles=(*.status)
  for filename in "${statusFiles[@]}"; do
    if [[ -r $filename ]]; then
      rm "${filename}"
    fi
  done
  popd >/dev/null 2>&1
  rmdir "${tempfolder}/FAIL"
}

cleanupOKCheckFiles() {
  pushd "${tempfolder}/OK" >/dev/null 2>&1
  okFiles=(*.duration)
  okCount=${#okFiles[@]}
  for filename in "${okFiles[@]}"; do
    if [[ -r $filename ]]; then
      rm "${filename}"
    fi
  done
  popd >/dev/null 2>&1
  rmdir "${tempfolder}/OK"
}


writeHeader() {
  echo '<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">'
  echo '<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/css/bootstrap.min.css" integrity="sha384-TX8t27EcRE3e/ihU7zmQxVncDAy5uIKz4rEkgIXeMed4M0jlfIDPvg6uqKI2xXr2" crossorigin="anonymous">'
  echo "<title>${title}</title>"
  echo "</head><body>"
  echo "<div class=container><div class=row><div class=col>"
  echo "<h1>${title}</h1>"
  echo "</div></div>"
  echo "<div class=row><div class=col>"
}

writeFooter() {
  echo "</div></div>"
  echo "<br>"
  echo "<div class=row><div class=col>"
  echo "<p class=small>Last check: "
  date
  echo "<br>Total duration: ${runtime} ms"
  echo "<br>Monitoring script by <a href='https://raymii.org/s/software/Bash_HTTP_Monitoring_Dashboard.html'>Remy van Elst</a>. License: GNU AGPLv3. "
  echo "<a href='https://github.com/raymiiorg/bash-http-monitoring'>Source code</a>"
  echo "</p>"
  echo "</div></div></div>"
  echo "</body></html>"
}


# script start 

if [[ "${cgi}" = true ]]; then
  printf "Content-type: text/html\n\n";
fi

# Total script duration timer
start=$(${datecmd} +%s%3N)

tmpdir=$(mktemp -d)
tempfolder=${tmpdir:-/tmp/statusmon/}

# try to create folders, if it fails, stop the script.
mkdir -p "${tempfolder}/OK" || exit 1
mkdir -p "${tempfolder}/FAIL" || exit 1

pushd "${tempfolder}" >/dev/null 2>&1

# Do the checks parallel
for key in "${!urls[@]}"
do
  value=${urls[$key]}
  if [[ "$(jobs | wc -l)" -ge ${maxConcurrentCurls} ]] ; then # run 12 curl commands at max parallel
      wait -n
  fi
  doRequest "$key" "$value" &
done
wait 

# Re-check any failed checks to prevent flapping
recheckFailedChecks

# header
writeHeader

# Failed checks, if any, go on top
writeFailedChecks

# Okay checks go below the failed checks
writeOkayChecks

# stop the total timer
end=$(${datecmd} +%s%3N)
runtime=$((end-start))

writeFooter

# Execute the callback for any failed URL's
doCallbackForFailedChecks

# Cleanup the status files
cleanupFailedCheckFiles
cleanupOKCheckFiles

popd >/dev/null 2>&1

rmdir "${tempfolder}"