silver-server-ops/scripts/check-cert-expiry.sh

#!/usr/bin/env bash

set -uo pipefail

CERT_DIR="${CERT_DIR:-/etc/letsencrypt/live}"
EXPIRY_DAYS="${EXPIRY_DAYS:-25}"
NTFY_URL="${NTFY_URL:-http://127.0.0.1:2586/certbot}"
NTFY_TITLE="${NTFY_TITLE:-SSL certificate warning}"
NTFY_PRIORITY="${NTFY_PRIORITY:-high}"
NTFY_TAGS="${NTFY_TAGS:-warning,lock}"
NTFY_TOKEN="${NTFY_TOKEN:-}"
ALERT_ON_NO_CERTS="${ALERT_ON_NO_CERTS:-true}"
CHECK_SERVED="${CHECK_SERVED:-true}"
SERVED_ADDR="${SERVED_ADDR:-127.0.0.1:443}"
SERVED_TIMEOUT="${SERVED_TIMEOUT:-10}"
HEALTHCHECK_URL="${HEALTHCHECK_URL:-}"

host="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo "unknown-host")"
now_epoch="$(date +%s)"
threshold_seconds=$((EXPIRY_DAYS * 86400))

send_alert() {
  local message="$1"
  local curl_args=(
    -fsS
    -H "Title: ${NTFY_TITLE}"
    -H "Priority: ${NTFY_PRIORITY}"
    -H "Tags: ${NTFY_TAGS}"
  )

  if [[ -n "${NTFY_TOKEN}" ]]; then
    curl_args+=(-H "Authorization: Bearer ${NTFY_TOKEN}")
  fi

  curl "${curl_args[@]}" --data-binary "${message}" "${NTFY_URL}" >/dev/null
}

# Expiry of the certificate nginx actually serves for a domain, via SNI on the
# loopback listener. stdin is fed from echo so that s_client closes the
# connection and, more importantly, does not consume the cert list the main
# loop is reading from its own stdin.
#
# nginx completes the TLS handshake with a fallback vhost certificate when SNI
# matches no server block, so the served certificate is only trustworthy for
# this comparison when its SAN list actually covers the domain. Anything else
# is reported as "not served" rather than compared against the wrong lineage.
served_expiry_epoch() {
  local domain="$1" pem sans enddate

  pem="$(echo | timeout "${SERVED_TIMEOUT}" openssl s_client \
    -connect "${SERVED_ADDR}" -servername "${domain}" 2>/dev/null \
    | openssl x509 2>/dev/null)"
  [[ -z "${pem}" ]] && return 1

  sans="$(printf '%s\n' "${pem}" | openssl x509 -noout -ext subjectAltName 2>/dev/null \
    | tr ',' '\n' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"
  grep -Fxq "DNS:${domain}" <<<"${sans}" || return 1

  enddate="$(printf '%s\n' "${pem}" | openssl x509 -noout -enddate 2>/dev/null)"
  date -d "${enddate#notAfter=}" +%s 2>/dev/null
}

# Dead man's switch. ntfy reports what the check found; this reports that the
# check ran at all. Without it, a dead cron, a failed boot or a bad chmod all
# look exactly like a healthy fleet.
ping_healthcheck() {
  local suffix="${1:-}"

  [[ -z "${HEALTHCHECK_URL}" ]] && return 0

  curl -fsS -m 10 -o /dev/null "${HEALTHCHECK_URL}${suffix}" \
    || echo "Failed to ping healthcheck at ${HEALTHCHECK_URL}${suffix}" >&2
}

fail() {
  local message="$1"
  send_alert "${message}" || echo "Failed to send ntfy alert to ${NTFY_URL}" >&2
  ping_healthcheck "/fail"
  echo "${message}" >&2
  exit 1
}

if ! command -v openssl >/dev/null 2>&1; then
  fail "SSL certificate check failed on ${host}: openssl is not installed."
fi

if ! command -v curl >/dev/null 2>&1; then
  echo "SSL certificate check failed on ${host}: curl is not installed." >&2
  exit 1
fi

if [[ ! -d "${CERT_DIR}" ]]; then
  fail "SSL certificate check failed on ${host}: certificate directory ${CERT_DIR} does not exist."
fi

warnings=()
cert_count=0

while IFS= read -r -d '' cert_path; do
  cert_count=$((cert_count + 1))
  cert_name="$(basename "$(dirname "${cert_path}")")"

  enddate_line="$(openssl x509 -in "${cert_path}" -noout -enddate 2>/dev/null)"
  if [[ -z "${enddate_line}" ]]; then
    warnings+=("${cert_name}: could not read expiry date from ${cert_path}")
    continue
  fi

  not_after="${enddate_line#notAfter=}"
  expiry_epoch="$(date -d "${not_after}" +%s 2>/dev/null)"
  if [[ -z "${expiry_epoch}" ]]; then
    warnings+=("${cert_name}: could not parse expiry date '${not_after}'")
    continue
  fi

  seconds_left=$((expiry_epoch - now_epoch))
  days_left=$((seconds_left / 86400))

  # certbot writes the new file, but nginx keeps serving the old certificate
  # from memory until it is reloaded. Comparing disk against what is served is
  # the only way to catch a renewal hook that stopped firing.
  if [[ "${CHECK_SERVED}" == "true" ]]; then
    if served_epoch="$(served_expiry_epoch "${cert_name}")" && [[ -n "${served_epoch}" ]]; then
      if (( served_epoch < expiry_epoch )); then
        served_days=$(( (served_epoch - now_epoch) / 86400 ))
        warnings+=("${cert_name}: renewed on disk but nginx is still serving the previous certificate (served copy expires in ${served_days} day(s)). Reload nginx.")
      fi
    fi
  fi

  if (( seconds_left < 0 )); then
    warnings+=("${cert_name}: EXPIRED on ${not_after}")
  elif ! openssl x509 -checkend "${threshold_seconds}" -noout -in "${cert_path}" >/dev/null 2>&1; then
    warnings+=("${cert_name}: expires in ${days_left} day(s), on ${not_after}")
  fi
done < <(find "${CERT_DIR}" -mindepth 2 -maxdepth 2 \( -type f -o -type l \) -name cert.pem -print0)

if (( cert_count == 0 )); then
  if [[ "${ALERT_ON_NO_CERTS}" == "true" ]]; then
    fail "SSL certificate check found no cert.pem files under ${CERT_DIR} on ${host}."
  fi

  echo "No certificates found under ${CERT_DIR}; no alert sent."
  ping_healthcheck
  exit 0
fi

if (( ${#warnings[@]} > 0 )); then
  message="SSL certificate expiry warning on ${host}. Threshold: ${EXPIRY_DAYS} day(s).

$(printf '%s\n' "${warnings[@]}")"

  send_alert "${message}" || echo "Failed to send ntfy alert to ${NTFY_URL}" >&2
  ping_healthcheck "/fail"
  echo "${message}" >&2
  exit 1
fi

echo "All ${cert_count} certificate(s) are valid for more than ${EXPIRY_DAYS} day(s)."
ping_healthcheck