diff --git a/scripts/worker.sh b/scripts/worker.sh index 1511d7e..d0c57cf 100644 --- a/scripts/worker.sh +++ b/scripts/worker.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -PLUGIN_DIR="$(cd "$(dirname "$0")/.." && pwd)" +PLUGIN_DIR="${PLUGIN_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" DATA_DIR="$PLUGIN_DIR/data" JOB_DIR="$DATA_DIR/jobs/pending" PROCESSING_DIR="$DATA_DIR/jobs/processing" @@ -10,15 +10,54 @@ WORKER_LOCK_DIR="$DATA_DIR/worker.lock" DB_LOCK_DIR="$DATA_DIR/lock" PID_DIR="$DATA_DIR/pids" CANCEL_DIR="$DATA_DIR/cancel" +WORKER_LOCK_STALE_SECONDS="${WORKER_LOCK_STALE_SECONDS:-300}" mkdir -p "$JOB_DIR" "$PROCESSING_DIR" "$DONE_DIR" "$PID_DIR" "$CANCEL_DIR" "$DB_LOCK_DIR" -if ! mkdir "$WORKER_LOCK_DIR" 2>/dev/null; then +acquire_worker_lock() { + if mkdir "$WORKER_LOCK_DIR" 2>/dev/null; then + printf '%s' "$$" > "$WORKER_LOCK_DIR/pid" 2>/dev/null || true + return 0 + fi + + local existing_pid="" lock_mtime=0 now lock_age=0 + if [ -r "$WORKER_LOCK_DIR/pid" ]; then + existing_pid="$(cat "$WORKER_LOCK_DIR/pid" 2>/dev/null || true)" + fi + + if [ -n "$existing_pid" ]; then + if kill -0 "$existing_pid" 2>/dev/null; then + echo "alt-mysql worker: lock held by running pid $existing_pid, exiting" >&2 + return 1 + fi + else + lock_mtime="$(stat -c '%Y' "$WORKER_LOCK_DIR" 2>/dev/null || stat -f '%m' "$WORKER_LOCK_DIR" 2>/dev/null || echo 0)" + now="$(date +%s)" + if [ "$lock_mtime" -gt 0 ]; then + lock_age=$((now - lock_mtime)) + fi + if [ "$lock_age" -lt "$WORKER_LOCK_STALE_SECONDS" ]; then + echo "alt-mysql worker: lock present without a live pid but younger than ${WORKER_LOCK_STALE_SECONDS}s (age ${lock_age}s), exiting" >&2 + return 1 + fi + fi + + echo "alt-mysql worker: reclaiming stale lock (pid=${existing_pid:-none}, age=${lock_age}s)" >&2 + rm -rf "$WORKER_LOCK_DIR" + if mkdir "$WORKER_LOCK_DIR" 2>/dev/null; then + printf '%s' "$$" > "$WORKER_LOCK_DIR/pid" 2>/dev/null || true + return 0 + fi + + return 1 +} + +if ! acquire_worker_lock; then exit 0 fi cleanup() { - rmdir "$WORKER_LOCK_DIR" 2>/dev/null || true + rm -rf "$WORKER_LOCK_DIR" 2>/dev/null || true } trap cleanup EXIT diff --git a/tests/worker_lock_test.sh b/tests/worker_lock_test.sh new file mode 100755 index 0000000..d63100a --- /dev/null +++ b/tests/worker_lock_test.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -euo pipefail + +PLUGIN_DIR_REAL="$(cd "$(dirname "$0")/.." && pwd)" +WORKER="$PLUGIN_DIR_REAL/scripts/worker.sh" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +backdate() { + local path="$1" seconds_ago="$2" + local ts + ts="$(date -v-"${seconds_ago}"S +%Y%m%d%H%M.%S 2>/dev/null || date -d "-${seconds_ago} seconds" +%Y%m%d%H%M.%S)" + touch -t "$ts" "$path" +} + +run_worker() { + PLUGIN_DIR="$TMP_DIR" bash "$WORKER" +} + +# --- Scenario 1: no existing lock - worker acquires it, runs, and cleans up normally --- +OUTPUT1="$(run_worker 2>&1)" +[ ! -e "$TMP_DIR/data/worker.lock" ] \ + || fail "worker lock should be removed after a normal run with an empty queue, output: $OUTPUT1" + +# --- Scenario 2: lock genuinely held by a live process - worker must back off, not reclaim --- +mkdir -p "$TMP_DIR/data" +mkdir "$TMP_DIR/data/worker.lock" +printf '%s' "$$" > "$TMP_DIR/data/worker.lock/pid" +OUTPUT2="$(run_worker 2>&1)" +echo "$OUTPUT2" | grep -qi "held by running pid" \ + || fail "worker must log that the lock is held by a live pid, got: $OUTPUT2" +[ -d "$TMP_DIR/data/worker.lock" ] \ + || fail "worker must not remove a lock held by a genuinely live process" +rm -rf "$TMP_DIR/data/worker.lock" + +# --- Scenario 3: lock left behind by a crashed worker (dead pid) - must be reclaimed --- +mkdir -p "$TMP_DIR/data" +mkdir "$TMP_DIR/data/worker.lock" +printf '999999999' > "$TMP_DIR/data/worker.lock/pid" +OUTPUT3="$(run_worker 2>&1)" +echo "$OUTPUT3" | grep -qi "reclaiming stale lock" \ + || fail "worker must log that it is reclaiming a stale lock (dead pid), got: $OUTPUT3" +[ ! -e "$TMP_DIR/data/worker.lock" ] \ + || fail "worker lock should be removed again after successfully reclaiming and running, output: $OUTPUT3" + +# --- Scenario 4: lock with no pid file but old enough to be considered abandoned --- +mkdir -p "$TMP_DIR/data" +mkdir "$TMP_DIR/data/worker.lock" +backdate "$TMP_DIR/data/worker.lock" 301 +OUTPUT4="$(run_worker 2>&1)" +echo "$OUTPUT4" | grep -qi "reclaiming stale lock" \ + || fail "worker must reclaim an old lock with no pid file, got: $OUTPUT4" +[ ! -e "$TMP_DIR/data/worker.lock" ] \ + || fail "worker lock should be removed again after reclaiming an old pid-less lock, output: $OUTPUT4" + +# --- Scenario 5: lock with no pid file, but too young to be considered abandoned yet --- +mkdir -p "$TMP_DIR/data" +mkdir "$TMP_DIR/data/worker.lock" +OUTPUT5="$(run_worker 2>&1)" +echo "$OUTPUT5" | grep -qi "younger than" \ + || fail "worker must back off on a young pid-less lock instead of reclaiming it, got: $OUTPUT5" +[ -d "$TMP_DIR/data/worker.lock" ] \ + || fail "worker must not remove a young pid-less lock (might just be mid-acquisition)" +rm -rf "$TMP_DIR/data/worker.lock" + +echo "worker_lock_test: OK"