Fix worker.sh lock permanently stalling after a crash

The mkdir-based worker lock had no staleness detection at all, unlike
BackupQueueService::withQueueLock()'s 120s override for its own queue
lock. If the root cron-driven worker was ever killed ungracefully (OOM,
kill -9, reboot mid-job), the lock directory was left behind forever -
every subsequent cron tick would silently exit without processing any
job, so backups/restores would queue forever with no visible error.

Track the owning PID inside the lock directory and reclaim the lock
when that PID is no longer running, or when the lock has no PID file
and is older than WORKER_LOCK_STALE_SECONDS (default 300s). Log to
stderr on both the "held by a live worker" and "reclaiming stale lock"
paths instead of exiting silently either way. cleanup() now uses rm -rf
since the lock directory holds a pid file, not just an empty marker.

tests/worker_lock_test.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+set -euo pipefail
+
+PLUGIN_DIR_REAL="$(cd "$(dirname "$0")/.." && pwd)"
+WORKER="$PLUGIN_DIR_REAL/scripts/worker.sh"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+fail() {
+  echo "FAIL: $*" >&2
+  exit 1
+}
+
+backdate() {
+  local path="$1" seconds_ago="$2"
+  local ts
+  ts="$(date -v-"${seconds_ago}"S +%Y%m%d%H%M.%S 2>/dev/null || date -d "-${seconds_ago} seconds" +%Y%m%d%H%M.%S)"
+  touch -t "$ts" "$path"
+}
+
+run_worker() {
+  PLUGIN_DIR="$TMP_DIR" bash "$WORKER"
+}
+
+# --- Scenario 1: no existing lock - worker acquires it, runs, and cleans up normally ---
+OUTPUT1="$(run_worker 2>&1)"
+[ ! -e "$TMP_DIR/data/worker.lock" ] \
+  || fail "worker lock should be removed after a normal run with an empty queue, output: $OUTPUT1"
+
+# --- Scenario 2: lock genuinely held by a live process - worker must back off, not reclaim ---
+mkdir -p "$TMP_DIR/data"
+mkdir "$TMP_DIR/data/worker.lock"
+printf '%s' "$$" > "$TMP_DIR/data/worker.lock/pid"
+OUTPUT2="$(run_worker 2>&1)"
+echo "$OUTPUT2" | grep -qi "held by running pid" \
+  || fail "worker must log that the lock is held by a live pid, got: $OUTPUT2"
+[ -d "$TMP_DIR/data/worker.lock" ] \
+  || fail "worker must not remove a lock held by a genuinely live process"
+rm -rf "$TMP_DIR/data/worker.lock"
+
+# --- Scenario 3: lock left behind by a crashed worker (dead pid) - must be reclaimed ---
+mkdir -p "$TMP_DIR/data"
+mkdir "$TMP_DIR/data/worker.lock"
+printf '999999999' > "$TMP_DIR/data/worker.lock/pid"
+OUTPUT3="$(run_worker 2>&1)"
+echo "$OUTPUT3" | grep -qi "reclaiming stale lock" \
+  || fail "worker must log that it is reclaiming a stale lock (dead pid), got: $OUTPUT3"
+[ ! -e "$TMP_DIR/data/worker.lock" ] \
+  || fail "worker lock should be removed again after successfully reclaiming and running, output: $OUTPUT3"
+
+# --- Scenario 4: lock with no pid file but old enough to be considered abandoned ---
+mkdir -p "$TMP_DIR/data"
+mkdir "$TMP_DIR/data/worker.lock"
+backdate "$TMP_DIR/data/worker.lock" 301
+OUTPUT4="$(run_worker 2>&1)"
+echo "$OUTPUT4" | grep -qi "reclaiming stale lock" \
+  || fail "worker must reclaim an old lock with no pid file, got: $OUTPUT4"
+[ ! -e "$TMP_DIR/data/worker.lock" ] \
+  || fail "worker lock should be removed again after reclaiming an old pid-less lock, output: $OUTPUT4"
+
+# --- Scenario 5: lock with no pid file, but too young to be considered abandoned yet ---
+mkdir -p "$TMP_DIR/data"
+mkdir "$TMP_DIR/data/worker.lock"
+OUTPUT5="$(run_worker 2>&1)"
+echo "$OUTPUT5" | grep -qi "younger than" \
+  || fail "worker must back off on a young pid-less lock instead of reclaiming it, got: $OUTPUT5"
+[ -d "$TMP_DIR/data/worker.lock" ] \
+  || fail "worker must not remove a young pid-less lock (might just be mid-acquisition)"
+rm -rf "$TMP_DIR/data/worker.lock"
+
+echo "worker_lock_test: OK"