Files
wirecon/lxc/usr/local/bin/wirecon-watchdog.sh
T

116 lines
3.7 KiB
Bash
Executable File

#!/bin/bash
# wirecon-watchdog.sh — invoked periodically by wirecon-watchdog.timer.
#
# Checks handshake age and in-tunnel reachability. If either fails for
# FAIL_STREAK_THRESHOLD consecutive ticks, bounces the WG interface and
# logs the reason.
#
# All output goes to stdout/stderr; systemd routes it to journald under
# SYSLOG_IDENTIFIER=wirecon-watchdog.
set -uo pipefail
CONF="${WIRECON_CONF:-/etc/wirecon/wirecon.conf}"
# shellcheck source=/dev/null
[[ -r "$CONF" ]] && source "$CONF"
: "${WG_INTERFACE:=wg0}"
: "${PEER_IP:=10.42.0.1}"
: "${HANDSHAKE_MAX_AGE:=180}"
: "${PING_COUNT:=3}"
: "${PING_TIMEOUT:=2}"
: "${PING_LOSS_THRESHOLD:=3}"
: "${FAIL_STREAK_THRESHOLD:=2}"
: "${STATE_DIR:=/run/wirecon}"
mkdir -p "$STATE_DIR"
STREAK_FILE="$STATE_DIR/fail_streak"
LAST_RECONNECT_FILE="$STATE_DIR/last_reconnect"
# Structured single-line log. kv pairs => easy to grep / jq via journalctl.
log() {
local level="$1"; shift
printf 'level=%s ts=%s %s\n' "$level" "$(date -u +%FT%TZ)" "$*"
}
read_streak() {
[[ -r "$STREAK_FILE" ]] && cat "$STREAK_FILE" || echo 0
}
write_streak() { echo "$1" > "$STREAK_FILE"; }
handshake_age() {
# `wg show <iface> latest-handshakes` → "<peer_pubkey>\t<unix_ts>"
local ts now
ts=$(wg show "$WG_INTERFACE" latest-handshakes 2>/dev/null | awk 'NR==1{print $2}')
[[ -z "$ts" || "$ts" == "0" ]] && { echo -1; return; }
now=$(date +%s)
echo $((now - ts))
}
in_tunnel_ping() {
# returns number of lost pings (0..PING_COUNT)
local out loss
out=$(ping -I "$WG_INTERFACE" -c "$PING_COUNT" -W "$PING_TIMEOUT" -n "$PEER_IP" 2>&1) || true
loss=$(echo "$out" | awk -F',' '/packet loss/ {gsub(/[^0-9]/,"",$3); print $3}')
# loss is percent; convert to absolute count
[[ -z "$loss" ]] && { echo "$PING_COUNT"; return; }
echo $(( PING_COUNT * loss / 100 ))
}
reconnect() {
local reason="$1"
log warn "event=reconnect reason=\"$reason\" iface=$WG_INTERFACE"
date -u +%FT%TZ > "$LAST_RECONNECT_FILE"
# `wg-quick down` can fail if the iface is already gone — that's fine.
wg-quick down "$WG_INTERFACE" 2>&1 | sed 's/^/ down: /' || true
sleep 1
if wg-quick up "$WG_INTERFACE" 2>&1 | sed 's/^/ up: /'; then
log info "event=reconnect_ok iface=$WG_INTERFACE"
write_streak 0
else
log error "event=reconnect_failed iface=$WG_INTERFACE"
fi
}
# --- main check --------------------------------------------------------------
# 1. Is the interface even there?
if ! ip link show "$WG_INTERFACE" >/dev/null 2>&1; then
log error "event=iface_missing iface=$WG_INTERFACE"
reconnect "interface missing"
exit 0
fi
age=$(handshake_age)
lost=$(in_tunnel_ping)
streak=$(read_streak)
# One-shot MTU probe on the first-ever successful handshake. The apply
# helper short-circuits on its own marker; we only skip the fork here to
# avoid spawning a subshell every tick once the marker exists.
if [[ "$age" -ge 0 && ! -e /var/lib/wirecon/mtu-probed.flag ]]; then
( /usr/local/bin/wirecon-mtu-apply.sh >>/var/log/wirecon-mtu.log 2>&1 ) &
fi
reason=""
if [[ "$age" -lt 0 ]]; then
reason="no handshake ever"
elif [[ "$age" -gt "$HANDSHAKE_MAX_AGE" ]]; then
reason="handshake stale age=${age}s max=${HANDSHAKE_MAX_AGE}s"
elif [[ "$lost" -ge "$PING_LOSS_THRESHOLD" ]]; then
reason="ping loss=${lost}/${PING_COUNT}"
fi
if [[ -n "$reason" ]]; then
streak=$((streak + 1))
write_streak "$streak"
log warn "event=check_fail streak=$streak/$FAIL_STREAK_THRESHOLD handshake_age=${age}s ping_lost=${lost}/${PING_COUNT} reason=\"$reason\""
if [[ "$streak" -ge "$FAIL_STREAK_THRESHOLD" ]]; then
reconnect "$reason (streak=$streak)"
fi
else
[[ "$streak" -ne 0 ]] && log info "event=recovered streak_was=$streak handshake_age=${age}s"
write_streak 0
log info "event=ok handshake_age=${age}s ping_lost=${lost}/${PING_COUNT}"
fi