diff --git a/README.md b/README.md index 838aeee..76e91cd 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,30 @@ CERT_DIR=/etc/letsencrypt/live EXPIRY_DAYS=25 NTFY_URL=http://127.0.0.1:2586/certbot ALERT_ON_NO_CERTS=true +CHECK_SERVED=true +SERVED_ADDR=127.0.0.1:443 +SERVED_TIMEOUT=10 ``` +### Served certificate check + +Certbot writes a renewed certificate to disk, but nginx keeps serving the old +one from memory until it is reloaded. A check that only reads +`/etc/letsencrypt/live` reports everything as healthy while browsers are being +handed an expired certificate. + +With `CHECK_SERVED=true` the script also opens a TLS connection to +`SERVED_ADDR` using each lineage's name as the SNI hostname and compares the +served expiry against the file on disk. It warns only when the served +certificate expires earlier than the one on disk, which is the signature of a +renewal hook that stopped firing. + +nginx completes a handshake with a fallback vhost certificate when SNI matches +no server block, so the comparison is skipped unless the served certificate's +SAN list actually covers that domain. A lineage that nginx no longer serves is +therefore skipped rather than compared against an unrelated certificate. Set +`CHECK_SERVED=false` to disable the check. + ### Overrides ```bash diff --git a/scripts/check-cert-expiry.sh b/scripts/check-cert-expiry.sh index 9a6400c..a0ea392 100644 --- a/scripts/check-cert-expiry.sh +++ b/scripts/check-cert-expiry.sh @@ -10,6 +10,9 @@ NTFY_PRIORITY="${NTFY_PRIORITY:-high}" NTFY_TAGS="${NTFY_TAGS:-warning,lock}" NTFY_TOKEN="${NTFY_TOKEN:-}" ALERT_ON_NO_CERTS="${ALERT_ON_NO_CERTS:-true}" +CHECK_SERVED="${CHECK_SERVED:-true}" +SERVED_ADDR="${SERVED_ADDR:-127.0.0.1:443}" +SERVED_TIMEOUT="${SERVED_TIMEOUT:-10}" host="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo "unknown-host")" now_epoch="$(date +%s)" @@ -31,6 +34,31 @@ send_alert() { curl "${curl_args[@]}" --data-binary "${message}" "${NTFY_URL}" >/dev/null } +# Expiry of the certificate nginx actually serves for a domain, via SNI on the +# loopback listener. stdin is fed from echo so that s_client closes the +# connection and, more importantly, does not consume the cert list the main +# loop is reading from its own stdin. +# +# nginx completes the TLS handshake with a fallback vhost certificate when SNI +# matches no server block, so the served certificate is only trustworthy for +# this comparison when its SAN list actually covers the domain. Anything else +# is reported as "not served" rather than compared against the wrong lineage. +served_expiry_epoch() { + local domain="$1" pem sans enddate + + pem="$(echo | timeout "${SERVED_TIMEOUT}" openssl s_client \ + -connect "${SERVED_ADDR}" -servername "${domain}" 2>/dev/null \ + | openssl x509 2>/dev/null)" + [[ -z "${pem}" ]] && return 1 + + sans="$(printf '%s\n' "${pem}" | openssl x509 -noout -ext subjectAltName 2>/dev/null \ + | tr ',' '\n' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')" + grep -Fxq "DNS:${domain}" <<<"${sans}" || return 1 + + enddate="$(printf '%s\n' "${pem}" | openssl x509 -noout -enddate 2>/dev/null)" + date -d "${enddate#notAfter=}" +%s 2>/dev/null +} + fail() { local message="$1" send_alert "${message}" || echo "Failed to send ntfy alert to ${NTFY_URL}" >&2 @@ -74,6 +102,18 @@ while IFS= read -r -d '' cert_path; do seconds_left=$((expiry_epoch - now_epoch)) days_left=$((seconds_left / 86400)) + # certbot writes the new file, but nginx keeps serving the old certificate + # from memory until it is reloaded. Comparing disk against what is served is + # the only way to catch a renewal hook that stopped firing. + if [[ "${CHECK_SERVED}" == "true" ]]; then + if served_epoch="$(served_expiry_epoch "${cert_name}")" && [[ -n "${served_epoch}" ]]; then + if (( served_epoch < expiry_epoch )); then + served_days=$(( (served_epoch - now_epoch) / 86400 )) + warnings+=("${cert_name}: renewed on disk but nginx is still serving the previous certificate (served copy expires in ${served_days} day(s)). Reload nginx.") + fi + fi + fi + if (( seconds_left < 0 )); then warnings+=("${cert_name}: EXPIRED on ${not_after}") elif ! openssl x509 -checkend "${threshold_seconds}" -noout -in "${cert_path}" >/dev/null 2>&1; then