116 lines
3.7 KiB
Bash
Executable File
116 lines
3.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# wirecon-watchdog.sh — invoked periodically by wirecon-watchdog.timer.
|
|
#
|
|
# Checks handshake age and in-tunnel reachability. If either fails for
|
|
# FAIL_STREAK_THRESHOLD consecutive ticks, bounces the WG interface and
|
|
# logs the reason.
|
|
#
|
|
# All output goes to stdout/stderr; systemd routes it to journald under
|
|
# SYSLOG_IDENTIFIER=wirecon-watchdog.
|
|
|
|
set -uo pipefail
|
|
|
|
CONF="${WIRECON_CONF:-/etc/wirecon/wirecon.conf}"
|
|
# shellcheck source=/dev/null
|
|
[[ -r "$CONF" ]] && source "$CONF"
|
|
|
|
: "${WG_INTERFACE:=wg0}"
|
|
: "${PEER_IP:=10.42.0.1}"
|
|
: "${HANDSHAKE_MAX_AGE:=180}"
|
|
: "${PING_COUNT:=3}"
|
|
: "${PING_TIMEOUT:=2}"
|
|
: "${PING_LOSS_THRESHOLD:=3}"
|
|
: "${FAIL_STREAK_THRESHOLD:=2}"
|
|
: "${STATE_DIR:=/run/wirecon}"
|
|
|
|
mkdir -p "$STATE_DIR"
|
|
STREAK_FILE="$STATE_DIR/fail_streak"
|
|
LAST_RECONNECT_FILE="$STATE_DIR/last_reconnect"
|
|
|
|
# Structured single-line log. kv pairs => easy to grep / jq via journalctl.
|
|
log() {
|
|
local level="$1"; shift
|
|
printf 'level=%s ts=%s %s\n' "$level" "$(date -u +%FT%TZ)" "$*"
|
|
}
|
|
|
|
read_streak() {
|
|
[[ -r "$STREAK_FILE" ]] && cat "$STREAK_FILE" || echo 0
|
|
}
|
|
write_streak() { echo "$1" > "$STREAK_FILE"; }
|
|
|
|
handshake_age() {
|
|
# `wg show <iface> latest-handshakes` → "<peer_pubkey>\t<unix_ts>"
|
|
local ts now
|
|
ts=$(wg show "$WG_INTERFACE" latest-handshakes 2>/dev/null | awk 'NR==1{print $2}')
|
|
[[ -z "$ts" || "$ts" == "0" ]] && { echo -1; return; }
|
|
now=$(date +%s)
|
|
echo $((now - ts))
|
|
}
|
|
|
|
in_tunnel_ping() {
|
|
# returns number of lost pings (0..PING_COUNT)
|
|
local out loss
|
|
out=$(ping -I "$WG_INTERFACE" -c "$PING_COUNT" -W "$PING_TIMEOUT" -n "$PEER_IP" 2>&1) || true
|
|
loss=$(echo "$out" | awk -F',' '/packet loss/ {gsub(/[^0-9]/,"",$3); print $3}')
|
|
# loss is percent; convert to absolute count
|
|
[[ -z "$loss" ]] && { echo "$PING_COUNT"; return; }
|
|
echo $(( PING_COUNT * loss / 100 ))
|
|
}
|
|
|
|
reconnect() {
|
|
local reason="$1"
|
|
log warn "event=reconnect reason=\"$reason\" iface=$WG_INTERFACE"
|
|
date -u +%FT%TZ > "$LAST_RECONNECT_FILE"
|
|
# `wg-quick down` can fail if the iface is already gone — that's fine.
|
|
wg-quick down "$WG_INTERFACE" 2>&1 | sed 's/^/ down: /' || true
|
|
sleep 1
|
|
if wg-quick up "$WG_INTERFACE" 2>&1 | sed 's/^/ up: /'; then
|
|
log info "event=reconnect_ok iface=$WG_INTERFACE"
|
|
write_streak 0
|
|
else
|
|
log error "event=reconnect_failed iface=$WG_INTERFACE"
|
|
fi
|
|
}
|
|
|
|
# --- main check --------------------------------------------------------------
|
|
|
|
# 1. Is the interface even there?
|
|
if ! ip link show "$WG_INTERFACE" >/dev/null 2>&1; then
|
|
log error "event=iface_missing iface=$WG_INTERFACE"
|
|
reconnect "interface missing"
|
|
exit 0
|
|
fi
|
|
|
|
age=$(handshake_age)
|
|
lost=$(in_tunnel_ping)
|
|
streak=$(read_streak)
|
|
|
|
# One-shot MTU probe on the first-ever successful handshake. The apply
|
|
# helper short-circuits on its own marker; we only skip the fork here to
|
|
# avoid spawning a subshell every tick once the marker exists.
|
|
if [[ "$age" -ge 0 && ! -e /var/lib/wirecon/mtu-probed.flag ]]; then
|
|
( /usr/local/bin/wirecon-mtu-apply.sh >>/var/log/wirecon-mtu.log 2>&1 ) &
|
|
fi
|
|
|
|
reason=""
|
|
if [[ "$age" -lt 0 ]]; then
|
|
reason="no handshake ever"
|
|
elif [[ "$age" -gt "$HANDSHAKE_MAX_AGE" ]]; then
|
|
reason="handshake stale age=${age}s max=${HANDSHAKE_MAX_AGE}s"
|
|
elif [[ "$lost" -ge "$PING_LOSS_THRESHOLD" ]]; then
|
|
reason="ping loss=${lost}/${PING_COUNT}"
|
|
fi
|
|
|
|
if [[ -n "$reason" ]]; then
|
|
streak=$((streak + 1))
|
|
write_streak "$streak"
|
|
log warn "event=check_fail streak=$streak/$FAIL_STREAK_THRESHOLD handshake_age=${age}s ping_lost=${lost}/${PING_COUNT} reason=\"$reason\""
|
|
if [[ "$streak" -ge "$FAIL_STREAK_THRESHOLD" ]]; then
|
|
reconnect "$reason (streak=$streak)"
|
|
fi
|
|
else
|
|
[[ "$streak" -ne 0 ]] && log info "event=recovered streak_was=$streak handshake_age=${age}s"
|
|
write_streak 0
|
|
log info "event=ok handshake_age=${age}s ping_lost=${lost}/${PING_COUNT}"
|
|
fi
|