#!/usr/bin/env bash set -uo pipefail CERT_DIR="${CERT_DIR:-/etc/letsencrypt/live}" EXPIRY_DAYS="${EXPIRY_DAYS:-25}" NTFY_URL="${NTFY_URL:-http://127.0.0.1:2586/certbot}" NTFY_TITLE="${NTFY_TITLE:-SSL certificate warning}" NTFY_PRIORITY="${NTFY_PRIORITY:-high}" NTFY_TAGS="${NTFY_TAGS:-warning,lock}" NTFY_TOKEN="${NTFY_TOKEN:-}" ALERT_ON_NO_CERTS="${ALERT_ON_NO_CERTS:-true}" CHECK_SERVED="${CHECK_SERVED:-true}" SERVED_ADDR="${SERVED_ADDR:-127.0.0.1:443}" SERVED_TIMEOUT="${SERVED_TIMEOUT:-10}" HEALTHCHECK_URL="${HEALTHCHECK_URL:-}" host="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo "unknown-host")" now_epoch="$(date +%s)" threshold_seconds=$((EXPIRY_DAYS * 86400)) send_alert() { local message="$1" local curl_args=( -fsS -H "Title: ${NTFY_TITLE}" -H "Priority: ${NTFY_PRIORITY}" -H "Tags: ${NTFY_TAGS}" ) if [[ -n "${NTFY_TOKEN}" ]]; then curl_args+=(-H "Authorization: Bearer ${NTFY_TOKEN}") fi curl "${curl_args[@]}" --data-binary "${message}" "${NTFY_URL}" >/dev/null } # Expiry of the certificate nginx actually serves for a domain, via SNI on the # loopback listener. stdin is fed from echo so that s_client closes the # connection and, more importantly, does not consume the cert list the main # loop is reading from its own stdin. # # nginx completes the TLS handshake with a fallback vhost certificate when SNI # matches no server block, so the served certificate is only trustworthy for # this comparison when its SAN list actually covers the domain. Anything else # is reported as "not served" rather than compared against the wrong lineage. served_expiry_epoch() { local domain="$1" pem sans enddate pem="$(echo | timeout "${SERVED_TIMEOUT}" openssl s_client \ -connect "${SERVED_ADDR}" -servername "${domain}" 2>/dev/null \ | openssl x509 2>/dev/null)" [[ -z "${pem}" ]] && return 1 sans="$(printf '%s\n' "${pem}" | openssl x509 -noout -ext subjectAltName 2>/dev/null \ | tr ',' '\n' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')" grep -Fxq "DNS:${domain}" <<<"${sans}" || return 1 enddate="$(printf '%s\n' "${pem}" | openssl x509 -noout -enddate 2>/dev/null)" date -d "${enddate#notAfter=}" +%s 2>/dev/null } # Dead man's switch. ntfy reports what the check found; this reports that the # check ran at all. Without it, a dead cron, a failed boot or a bad chmod all # look exactly like a healthy fleet. ping_healthcheck() { local suffix="${1:-}" [[ -z "${HEALTHCHECK_URL}" ]] && return 0 curl -fsS -m 10 -o /dev/null "${HEALTHCHECK_URL}${suffix}" \ || echo "Failed to ping healthcheck at ${HEALTHCHECK_URL}${suffix}" >&2 } fail() { local message="$1" send_alert "${message}" || echo "Failed to send ntfy alert to ${NTFY_URL}" >&2 ping_healthcheck "/fail" echo "${message}" >&2 exit 1 } if ! command -v openssl >/dev/null 2>&1; then fail "SSL certificate check failed on ${host}: openssl is not installed." fi if ! command -v curl >/dev/null 2>&1; then echo "SSL certificate check failed on ${host}: curl is not installed." >&2 exit 1 fi if [[ ! -d "${CERT_DIR}" ]]; then fail "SSL certificate check failed on ${host}: certificate directory ${CERT_DIR} does not exist." fi warnings=() cert_count=0 while IFS= read -r -d '' cert_path; do cert_count=$((cert_count + 1)) cert_name="$(basename "$(dirname "${cert_path}")")" enddate_line="$(openssl x509 -in "${cert_path}" -noout -enddate 2>/dev/null)" if [[ -z "${enddate_line}" ]]; then warnings+=("${cert_name}: could not read expiry date from ${cert_path}") continue fi not_after="${enddate_line#notAfter=}" expiry_epoch="$(date -d "${not_after}" +%s 2>/dev/null)" if [[ -z "${expiry_epoch}" ]]; then warnings+=("${cert_name}: could not parse expiry date '${not_after}'") continue fi seconds_left=$((expiry_epoch - now_epoch)) days_left=$((seconds_left / 86400)) # certbot writes the new file, but nginx keeps serving the old certificate # from memory until it is reloaded. Comparing disk against what is served is # the only way to catch a renewal hook that stopped firing. if [[ "${CHECK_SERVED}" == "true" ]]; then if served_epoch="$(served_expiry_epoch "${cert_name}")" && [[ -n "${served_epoch}" ]]; then if (( served_epoch < expiry_epoch )); then served_days=$(( (served_epoch - now_epoch) / 86400 )) warnings+=("${cert_name}: renewed on disk but nginx is still serving the previous certificate (served copy expires in ${served_days} day(s)). Reload nginx.") fi fi fi if (( seconds_left < 0 )); then warnings+=("${cert_name}: EXPIRED on ${not_after}") elif ! openssl x509 -checkend "${threshold_seconds}" -noout -in "${cert_path}" >/dev/null 2>&1; then warnings+=("${cert_name}: expires in ${days_left} day(s), on ${not_after}") fi done < <(find "${CERT_DIR}" -mindepth 2 -maxdepth 2 \( -type f -o -type l \) -name cert.pem -print0) if (( cert_count == 0 )); then if [[ "${ALERT_ON_NO_CERTS}" == "true" ]]; then fail "SSL certificate check found no cert.pem files under ${CERT_DIR} on ${host}." fi echo "No certificates found under ${CERT_DIR}; no alert sent." ping_healthcheck exit 0 fi if (( ${#warnings[@]} > 0 )); then message="SSL certificate expiry warning on ${host}. Threshold: ${EXPIRY_DAYS} day(s). $(printf '%s\n' "${warnings[@]}")" send_alert "${message}" || echo "Failed to send ntfy alert to ${NTFY_URL}" >&2 ping_healthcheck "/fail" echo "${message}" >&2 exit 1 fi echo "All ${cert_count} certificate(s) are valid for more than ${EXPIRY_DAYS} day(s)." ping_healthcheck