#!/bin/bash # wirecon-watchdog.sh — invoked periodically by wirecon-watchdog.timer. # # Checks handshake age and in-tunnel reachability. If either fails for # FAIL_STREAK_THRESHOLD consecutive ticks, bounces the WG interface and # logs the reason. # # All output goes to stdout/stderr; systemd routes it to journald under # SYSLOG_IDENTIFIER=wirecon-watchdog. set -uo pipefail CONF="${WIRECON_CONF:-/etc/wirecon/wirecon.conf}" # shellcheck source=/dev/null [[ -r "$CONF" ]] && source "$CONF" : "${WG_INTERFACE:=wg0}" : "${PEER_IP:=10.42.0.1}" : "${HANDSHAKE_MAX_AGE:=180}" : "${PING_COUNT:=3}" : "${PING_TIMEOUT:=2}" : "${PING_LOSS_THRESHOLD:=3}" : "${FAIL_STREAK_THRESHOLD:=2}" : "${STATE_DIR:=/run/wirecon}" mkdir -p "$STATE_DIR" STREAK_FILE="$STATE_DIR/fail_streak" LAST_RECONNECT_FILE="$STATE_DIR/last_reconnect" # Structured single-line log. kv pairs => easy to grep / jq via journalctl. log() { local level="$1"; shift printf 'level=%s ts=%s %s\n' "$level" "$(date -u +%FT%TZ)" "$*" } read_streak() { [[ -r "$STREAK_FILE" ]] && cat "$STREAK_FILE" || echo 0 } write_streak() { echo "$1" > "$STREAK_FILE"; } handshake_age() { # `wg show latest-handshakes` → "\t" local ts now ts=$(wg show "$WG_INTERFACE" latest-handshakes 2>/dev/null | awk 'NR==1{print $2}') [[ -z "$ts" || "$ts" == "0" ]] && { echo -1; return; } now=$(date +%s) echo $((now - ts)) } in_tunnel_ping() { # returns number of lost pings (0..PING_COUNT) local out loss out=$(ping -I "$WG_INTERFACE" -c "$PING_COUNT" -W "$PING_TIMEOUT" -n "$PEER_IP" 2>&1) || true loss=$(echo "$out" | awk -F',' '/packet loss/ {gsub(/[^0-9]/,"",$3); print $3}') # loss is percent; convert to absolute count [[ -z "$loss" ]] && { echo "$PING_COUNT"; return; } echo $(( PING_COUNT * loss / 100 )) } reconnect() { local reason="$1" log warn "event=reconnect reason=\"$reason\" iface=$WG_INTERFACE" date -u +%FT%TZ > "$LAST_RECONNECT_FILE" # `wg-quick down` can fail if the iface is already gone — that's fine. wg-quick down "$WG_INTERFACE" 2>&1 | sed 's/^/ down: /' || true sleep 1 if wg-quick up "$WG_INTERFACE" 2>&1 | sed 's/^/ up: /'; then log info "event=reconnect_ok iface=$WG_INTERFACE" write_streak 0 else log error "event=reconnect_failed iface=$WG_INTERFACE" fi } # --- main check -------------------------------------------------------------- # 1. Is the interface even there? if ! ip link show "$WG_INTERFACE" >/dev/null 2>&1; then log error "event=iface_missing iface=$WG_INTERFACE" reconnect "interface missing" exit 0 fi age=$(handshake_age) lost=$(in_tunnel_ping) streak=$(read_streak) reason="" if [[ "$age" -lt 0 ]]; then reason="no handshake ever" elif [[ "$age" -gt "$HANDSHAKE_MAX_AGE" ]]; then reason="handshake stale age=${age}s max=${HANDSHAKE_MAX_AGE}s" elif [[ "$lost" -ge "$PING_LOSS_THRESHOLD" ]]; then reason="ping loss=${lost}/${PING_COUNT}" fi if [[ -n "$reason" ]]; then streak=$((streak + 1)) write_streak "$streak" log warn "event=check_fail streak=$streak/$FAIL_STREAK_THRESHOLD handshake_age=${age}s ping_lost=${lost}/${PING_COUNT} reason=\"$reason\"" if [[ "$streak" -ge "$FAIL_STREAK_THRESHOLD" ]]; then reconnect "$reason (streak=$streak)" fi else [[ "$streak" -ne 0 ]] && log info "event=recovered streak_was=$streak handshake_age=${age}s" write_streak 0 log info "event=ok handshake_age=${age}s ping_lost=${lost}/${PING_COUNT}" fi