v2.5 通过镜像固化stateless ray pool

2025-12-29 15:54:13 +08:00
parent f02059126e
commit c06f61ed03
14 changed files with 560 additions and 96 deletions
--- a/src/mvp/docker-compose.yaml
+++ b/src/mvp/docker-compose.yaml
@@ -1,10 +1,12 @@
-version: "3.8"
-
 services:
  ray_head:
-    image: verlai/verl:sgl055.latest
+    build:
+      context: .
+      dockerfile: images/argus-ray-node/Dockerfile
+      args:
+        BASE_IMAGE: verlai/verl:sgl055.latest
+    image: argus/argus-ray-node:v2.5
    container_name: argus-ray-head
-    command: sleep infinity
    ports:
      - "8265:8265"
      - "8080:8080"
@@ -26,6 +28,10 @@ services:
    networks:
      - argus-ray-net
    environment:
+      ARGUS_ROLE: "head"
+      ARGUS_SHARED_ROOT: "/private"
+      ARGUS_CLUSTER_NAME: "argus-ray"
+      ARGUS_LOG_DIR: "/private/common/logs"
      HF_HOME: "/private/hf"
      HUGGINGFACE_HUB_CACHE: "/private/hf/hub"
      TRANSFORMERS_CACHE: "/private/hf/transformers"
@@ -33,9 +39,8 @@ services:
      PYTHONUNBUFFERED: "1"

  ray_worker_0:
-    image: verlai/verl:sgl055.latest
+    image: argus/argus-ray-node:v2.5
    container_name: argus-ray-worker-0
-    command: sleep infinity
    volumes:
      - ../../verl:/workspace/verl
      - ../../shared:/private
@@ -52,6 +57,11 @@ services:
      - argus-ray-net
    runtime: nvidia
    environment:
+      ARGUS_ROLE: "worker"
+      ARGUS_SHARED_ROOT: "/private"
+      ARGUS_CLUSTER_NAME: "argus-ray"
+      ARGUS_LOG_DIR: "/private/common/logs"
+      ARGUS_WORKER_RESOURCES_KV: "worker_node=100"
      NVIDIA_VISIBLE_DEVICES: "0,1,2,3"
      NVIDIA_DRIVER_CAPABILITIES: "all"
      HF_HOME: "/private/hf"
@@ -61,9 +71,8 @@ services:
      PYTHONUNBUFFERED: "1"

  ray_worker_1:
-    image: verlai/verl:sgl055.latest
+    image: argus/argus-ray-node:v2.5
    container_name: argus-ray-worker-1
-    command: sleep infinity
    volumes:
      - ../../verl:/workspace/verl
      - ../../shared:/private
@@ -80,6 +89,11 @@ services:
      - argus-ray-net
    runtime: nvidia
    environment:
+      ARGUS_ROLE: "worker"
+      ARGUS_SHARED_ROOT: "/private"
+      ARGUS_CLUSTER_NAME: "argus-ray"
+      ARGUS_LOG_DIR: "/private/common/logs"
+      ARGUS_WORKER_RESOURCES_KV: "worker_node=100"
      NVIDIA_VISIBLE_DEVICES: "4,5,6,7"
      NVIDIA_DRIVER_CAPABILITIES: "all"
      HF_HOME: "/private/hf"
--- a/src/mvp/images/argus-ray-node/Dockerfile
+++ b/src/mvp/images/argus-ray-node/Dockerfile
@@ -0,0 +1,26 @@
+ARG BASE_IMAGE=verlai/verl:sgl055.latest
+FROM ${BASE_IMAGE}
+
+SHELL ["/bin/bash", "-lc"]
+
+# Install supervisord (prefer pip to avoid relying on distro package manager).
+RUN python3 -m pip install --no-cache-dir supervisor
+
+RUN mkdir -p /opt/argus/py/argus/ray
+
+# Minimal embedded code for stateless pool (API code is intentionally excluded).
+COPY py/argus/__init__.py /opt/argus/py/argus/__init__.py
+COPY py/argus/ray/__init__.py /opt/argus/py/argus/ray/__init__.py
+COPY py/argus/ray/discovery.py /opt/argus/py/argus/ray/discovery.py
+COPY py/argus/ray/head_publisher.py /opt/argus/py/argus/ray/head_publisher.py
+COPY py/argus/ray/worker_watchdog.py /opt/argus/py/argus/ray/worker_watchdog.py
+
+COPY images/argus-ray-node/entrypoint.sh /usr/local/bin/argus-entrypoint.sh
+COPY images/argus-ray-node/argus-head-ray.sh /usr/local/bin/argus-head-ray.sh
+COPY images/argus-ray-node/argus-head-publisher.sh /usr/local/bin/argus-head-publisher.sh
+COPY images/argus-ray-node/argus-worker-watchdog.sh /usr/local/bin/argus-worker-watchdog.sh
+RUN chmod +x /usr/local/bin/argus-entrypoint.sh /usr/local/bin/argus-head-ray.sh /usr/local/bin/argus-head-publisher.sh /usr/local/bin/argus-worker-watchdog.sh
+
+ENV PYTHONUNBUFFERED=1
+
+ENTRYPOINT ["/usr/local/bin/argus-entrypoint.sh"]
--- a/src/mvp/images/argus-ray-node/argus-head-publisher.sh
+++ b/src/mvp/images/argus-ray-node/argus-head-publisher.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cluster_name="${ARGUS_CLUSTER_NAME:-argus-ray}"
+head_ip_file="${ARGUS_HEAD_IP_FILE:?missing ARGUS_HEAD_IP_FILE}"
+node_ip="${ARGUS_NODE_IP:?missing ARGUS_NODE_IP}"
+ray_port="${ARGUS_RAY_PORT:-6379}"
+dashboard_port="${ARGUS_DASHBOARD_PORT:-8265}"
+ttl_s="${ARGUS_TTL_S:-60}"
+refresh_s="${ARGUS_REFRESH_S:-10}"
+
+exec python3 -m argus.ray.head_publisher \
+  --cluster-name "${cluster_name}" \
+  --head-ip-file "${head_ip_file}" \
+  --head-ip "${node_ip}" \
+  --gcs-port "${ray_port}" \
+  --dashboard-port "${dashboard_port}" \
+  --ttl-s "${ttl_s}" \
+  --refresh-s "${refresh_s}"
+
--- a/src/mvp/images/argus-ray-node/argus-head-ray.sh
+++ b/src/mvp/images/argus-ray-node/argus-head-ray.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+node_ip="${ARGUS_NODE_IP:?missing ARGUS_NODE_IP}"
+ray_port="${ARGUS_RAY_PORT:-6379}"
+dashboard_port="${ARGUS_DASHBOARD_PORT:-8265}"
+
+export RAY_DISABLE_USAGE_STATS=1
+
+ray stop --force || true
+
+# Head should not run training workloads.
+exec ray start \
+  --head \
+  --node-ip-address="${node_ip}" \
+  --port="${ray_port}" \
+  --dashboard-host=0.0.0.0 \
+  --dashboard-port="${dashboard_port}" \
+  --num-cpus=0 \
+  --num-gpus=0 \
+  --disable-usage-stats \
+  --block \
+  ${ARGUS_RAY_EXTRA_ARGS:-}
+
--- a/src/mvp/images/argus-ray-node/argus-worker-watchdog.sh
+++ b/src/mvp/images/argus-ray-node/argus-worker-watchdog.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+head_ip_file="${ARGUS_HEAD_IP_FILE:?missing ARGUS_HEAD_IP_FILE}"
+node_ip="${ARGUS_NODE_IP:?missing ARGUS_NODE_IP}"
+poll_s="${ARGUS_POLL_S:-5}"
+
+resources_kv="${ARGUS_WORKER_RESOURCES_KV:-worker_node=100}"
+
+args=()
+IFS=',' read -r -a parts <<<"${resources_kv}"
+for kv in "${parts[@]}"; do
+  kv="$(echo "${kv}" | xargs || true)"
+  [[ -z "${kv}" ]] && continue
+  args+=( "--resources-kv" "${kv}" )
+done
+
+exec python3 -m argus.ray.worker_watchdog \
+  --head-ip-file "${head_ip_file}" \
+  --node-ip "${node_ip}" \
+  "${args[@]}" \
+  --poll-s "${poll_s}" \
+  --ray-start-block
+
--- a/src/mvp/images/argus-ray-node/entrypoint.sh
+++ b/src/mvp/images/argus-ray-node/entrypoint.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+role="${ARGUS_ROLE:-}"
+if [[ -z "${role}" ]]; then
+  echo "ERROR: ARGUS_ROLE must be set to head|worker" >&2
+  exit 1
+fi
+if [[ "${role}" != "head" && "${role}" != "worker" ]]; then
+  echo "ERROR: invalid ARGUS_ROLE=${role} (expected head|worker)" >&2
+  exit 1
+fi
+
+shared_root="${ARGUS_SHARED_ROOT:-/private}"
+cluster_name="${ARGUS_CLUSTER_NAME:-argus-ray}"
+head_ip_file="${ARGUS_HEAD_IP_FILE:-${shared_root}/ray/discovery/${cluster_name}/head.json}"
+
+ray_port="${ARGUS_RAY_PORT:-6379}"
+dashboard_port="${ARGUS_DASHBOARD_PORT:-8265}"
+
+ttl_s="${ARGUS_TTL_S:-60}"
+refresh_s="${ARGUS_REFRESH_S:-10}"
+poll_s="${ARGUS_POLL_S:-5}"
+
+log_dir="${ARGUS_LOG_DIR:-${shared_root}/common/logs}"
+mkdir -p "${log_dir}"
+
+node_ip="${ARGUS_NODE_IP:-}"
+if [[ -z "${node_ip}" ]]; then
+  # Prefer the first IP returned by `hostname -i`.
+  node_ip="$(hostname -i | awk '{print $1}')"
+fi
+if [[ -z "${node_ip}" ]]; then
+  echo "ERROR: failed to determine container IP; set ARGUS_NODE_IP" >&2
+  exit 1
+fi
+
+export PYTHONPATH="/opt/argus/py:${PYTHONPATH:-}"
+export ARGUS_SHARED_ROOT="${shared_root}"
+export ARGUS_CLUSTER_NAME="${cluster_name}"
+export ARGUS_HEAD_IP_FILE="${head_ip_file}"
+export ARGUS_NODE_IP="${node_ip}"
+export ARGUS_RAY_PORT="${ray_port}"
+export ARGUS_DASHBOARD_PORT="${dashboard_port}"
+export ARGUS_TTL_S="${ttl_s}"
+export ARGUS_REFRESH_S="${refresh_s}"
+export ARGUS_POLL_S="${poll_s}"
+export ARGUS_LOG_DIR="${log_dir}"
+
+supervisor_conf="/tmp/supervisord.conf"
+cat >"${supervisor_conf}" <<'CONF'
+[supervisord]
+nodaemon=true
+pidfile=/tmp/supervisord.pid
+logfile=/dev/null
+childlogdir=/tmp
+user=root
+
+[supervisorctl]
+serverurl=unix:///tmp/supervisor.sock
+
+[unix_http_server]
+file=/tmp/supervisor.sock
+chmod=0700
+CONF
+
+append_program() {
+  local name="$1"
+  local cmd="$2"
+  local stdout="$3"
+  local stderr="$4"
+  cat >>"${supervisor_conf}" <<CONF
+
+[program:${name}]
+command=${cmd}
+autorestart=true
+startsecs=1
+startretries=999
+stdout_logfile=${stdout}
+stdout_logfile_maxbytes=0
+stderr_logfile=${stderr}
+stderr_logfile_maxbytes=0
+CONF
+}
+
+if [[ "${role}" == "head" ]]; then
+  append_program "argus_head_ray" "/usr/local/bin/argus-head-ray.sh" "/dev/fd/1" "/dev/fd/2"
+  append_program "argus_head_publisher" "/usr/local/bin/argus-head-publisher.sh" "${log_dir}/argus_head_publisher.log" "${log_dir}/argus_head_publisher.err.log"
+else
+  append_program "argus_worker_watchdog" "/usr/local/bin/argus-worker-watchdog.sh" "${log_dir}/argus_worker_watchdog.log" "${log_dir}/argus_worker_watchdog.err.log"
+fi
+
+exec supervisord -n -c "${supervisor_conf}"
--- a/src/mvp/py/argus/ray/worker_watchdog.py
+++ b/src/mvp/py/argus/ray/worker_watchdog.py
@@ -2,6 +2,7 @@ from __future__ import annotations

 import argparse
 import json
+import os
 import subprocess
 import time
 from dataclasses import dataclass
@@ -22,10 +23,12 @@ def _ray_stop_cmd() -> list[str]:
    return ["ray", "stop", "--force"]


-def _ray_start_cmd(*, head_addr: str, node_ip: str | None, resources_json: str) -> list[str]:
+def _ray_start_cmd(*, head_addr: str, node_ip: str | None, resources_json: str, block: bool) -> list[str]:
    argv = ["ray", "start", f"--address={head_addr}", f"--resources={resources_json}", "--disable-usage-stats"]
    if node_ip:
        argv.append(f"--node-ip-address={node_ip}")
+    if block:
+        argv.append("--block")
    return argv


@@ -36,13 +39,54 @@ class Watchdog:
    resources_json: str
    poll_s: int
    runner: Runner
+    ray_start_block: bool

    _current_head_addr: str | None = None
+    _ray_proc: subprocess.Popen[bytes] | None = None
+
+    def _kill_ray_proc(self) -> None:
+        p = self._ray_proc
+        self._ray_proc = None
+        if not p:
+            return
+        try:
+            p.terminate()
+        except Exception:
+            pass
+        try:
+            p.wait(timeout=10)
+        except Exception:
+            try:
+                p.kill()
+            except Exception:
+                pass
+            try:
+                p.wait(timeout=5)
+            except Exception:
+                pass

    def _restart_ray(self, desired: str) -> None:
        # Best-effort stop then start.
+        self._kill_ray_proc()
        self.runner(_ray_stop_cmd())
-        self.runner(_ray_start_cmd(head_addr=desired, node_ip=self.node_ip, resources_json=self.resources_json))
+        if self.ray_start_block:
+            argv = _ray_start_cmd(
+                head_addr=desired,
+                node_ip=self.node_ip,
+                resources_json=self.resources_json,
+                block=True,
+            )
+            # Use Popen so watchdog can notice exits and reconnect if needed.
+            self._ray_proc = subprocess.Popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        else:
+            self.runner(
+                _ray_start_cmd(
+                    head_addr=desired,
+                    node_ip=self.node_ip,
+                    resources_json=self.resources_json,
+                    block=False,
+                )
+            )
        self._current_head_addr = desired

    def tick_once(self) -> HeadRecord | None:
@@ -52,6 +96,12 @@ class Watchdog:
        desired = rec.head_addr()
        if self._current_head_addr != desired:
            self._restart_ray(desired)
+            return rec
+
+        # If we're in block mode and the ray process died, restart it against the same head.
+        if self.ray_start_block and self._ray_proc is not None:
+            if self._ray_proc.poll() is not None:
+                self._restart_ray(desired)
        return rec

    def run_forever(self) -> None:
@@ -75,6 +125,7 @@ def main(argv: list[str] | None = None) -> int:
        help='Repeatable k=v resources, e.g. --resources-kv worker_node=100',
    )
    ap.add_argument("--poll-s", type=int, default=5)
+    ap.add_argument("--ray-start-block", action="store_true", help="Run `ray start ... --block` and self-heal on exit.")
    ap.add_argument("--once", action="store_true", help="Run one tick then exit (for testing/debug).")
    args = ap.parse_args(argv)

@@ -91,12 +142,17 @@ def main(argv: list[str] | None = None) -> int:
                raise SystemExit(f"invalid --resources-kv key: {item!r}")
            resources[k] = float(v)

+    node_ip = args.node_ip
+    if node_ip in (None, ""):
+        node_ip = os.environ.get("ARGUS_NODE_IP") or None
+
    wd = Watchdog(
        head_ip_file=args.head_ip_file,
-        node_ip=args.node_ip,
+        node_ip=node_ip,
        resources_json=json.dumps(resources),
        poll_s=int(args.poll_s),
        runner=_default_runner,
+        ray_start_block=bool(args.ray_start_block),
    )
    if args.once:
        wd.tick_once()
--- a/src/mvp/py/tests/test_worker_watchdog.py
+++ b/src/mvp/py/tests/test_worker_watchdog.py
@@ -21,6 +21,7 @@ def test_watchdog_restarts_on_first_seen_and_on_change(tmp_path: Path):
        resources_json=json.dumps({"worker_node": 100}),
        poll_s=1,
        runner=runner,
+        ray_start_block=False,
    )

    write_head_record_atomic(str(head_file), build_head_record(cluster_name="c", head_ip="1.1.1.1"))
@@ -40,6 +41,72 @@ def test_watchdog_restarts_on_first_seen_and_on_change(tmp_path: Path):
    assert any(c[1] == "start" for c in calls)


+def test_watchdog_block_mode_restarts_when_ray_exits(monkeypatch, tmp_path: Path):
+    import argus.ray.worker_watchdog as mod
+
+    head_file = tmp_path / "head.json"
+    write_head_record_atomic(str(head_file), build_head_record(cluster_name="c", head_ip="1.1.1.1"))
+
+    calls: list[list[str]] = []
+    popen_argvs: list[list[str]] = []
+
+    class DummyProc:
+        def __init__(self, argv: list[str]):
+            self.argv = argv
+            self._poll: int | None = None
+            self.terminated = False
+            self.killed = False
+
+        def poll(self):
+            return self._poll
+
+        def terminate(self):
+            self.terminated = True
+
+        def wait(self, timeout: float | None = None):
+            # Force kill path to be exercised.
+            raise TimeoutError("timeout")
+
+        def kill(self):
+            self.killed = True
+
+    proc_holder: dict[str, DummyProc] = {}
+
+    def fake_popen(argv, stdout=None, stderr=None):
+        popen_argvs.append(list(argv))
+        p = DummyProc(list(argv))
+        proc_holder["p"] = p
+        return p
+
+    def runner(argv: list[str]) -> int:
+        calls.append(argv)
+        return 0
+
+    monkeypatch.setattr(mod.subprocess, "Popen", fake_popen)
+
+    wd = Watchdog(
+        head_ip_file=str(head_file),
+        node_ip="10.0.0.2",
+        resources_json=json.dumps({"worker_node": 100}),
+        poll_s=1,
+        runner=runner,
+        ray_start_block=True,
+    )
+
+    # First tick: should start ray with --block.
+    assert wd.tick_once() is not None
+    assert any(c[1] == "stop" for c in calls)
+    assert len(popen_argvs) == 1
+    assert "--block" in popen_argvs[0]
+
+    # Simulate ray process exit; next tick should restart (stop + new popen).
+    proc_holder["p"]._poll = 1
+    calls.clear()
+    wd.tick_once()
+    assert any(c[1] == "stop" for c in calls)
+    assert len(popen_argvs) == 2
+
+
 def test_watchdog_main_once_invokes_runner(monkeypatch, tmp_path: Path):
    from argus.ray import worker_watchdog as mod

--- a/src/mvp/scripts/01_up.sh
+++ b/src/mvp/scripts/01_up.sh
@@ -10,7 +10,12 @@ if [[ "${SKIP_CLEANUP_V1:-0}" != "1" ]]; then
 fi

 echo "[host] docker compose up -d (mvp)"
-dc up -d
+BUILD="${BUILD:-1}"
+if [[ "${BUILD}" == "1" ]]; then
+  dc up -d --build
+else
+  dc up -d
+fi

 echo "[host] containers:"
 docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | (head -n 1 && grep -E "argus-ray-") || true
--- a/src/mvp/scripts/20_start_head.sh
+++ b/src/mvp/scripts/20_start_head.sh
@@ -5,14 +5,17 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=lib.sh
 source "${SCRIPT_DIR}/lib.sh"

-HEAD_IP="$(container_ip "${HEAD_CONTAINER}")"
+echo "[head] restart head container (ray + publisher are supervised in-container): ${HEAD_CONTAINER}"
+docker restart "${HEAD_CONTAINER}" >/dev/null

-echo "[head] ray stop (best-effort)"
-dexec "${HEAD_CONTAINER}" bash -lc "ray stop --force || true"
+echo "[head] wait ray dashboard ready (best-effort)"
+for i in $(seq 1 60); do
+  if curl -sS -m 2 "${RAY_DASHBOARD_ADDR}" >/dev/null 2>&1; then
+    echo "[head] dashboard ready: ${RAY_DASHBOARD_ADDR}"
+    break
+  fi
+  sleep 2
+done

-echo "[head] start ray head (CPU=0 GPU=0): ${HEAD_IP}"
-dexec "${HEAD_CONTAINER}" bash -lc "ray start --head --node-ip-address='${HEAD_IP}' --port=6379 --dashboard-host=0.0.0.0 --dashboard-port=8265 --num-cpus=0 --num-gpus=0 --disable-usage-stats"
-
-echo "[head] ray status"
+echo "[head] ray status (best-effort)"
 dexec "${HEAD_CONTAINER}" bash -lc "ray status || true"
-
--- a/src/mvp/scripts/22_start_head_discovery.sh
+++ b/src/mvp/scripts/22_start_head_discovery.sh
@@ -7,38 +7,6 @@ source "${SCRIPT_DIR}/lib.sh"

 CLUSTER_NAME="${CLUSTER_NAME:-argus-ray}"
 HEAD_IP_FILE="${HEAD_IP_FILE:-${SHARED_ROOT}/ray/discovery/${CLUSTER_NAME}/head.json}"
-TTL_S="${TTL_S:-60}"
-REFRESH_S="${REFRESH_S:-10}"
-
-# PID file must be container-local to avoid conflicts if /private is shared across containers.
-PID_PATH="${PID_PATH:-/tmp/argus_head_publisher.pid}"
-LOG_PATH="${LOG_PATH:-${SHARED_ROOT}/common/logs/argus_head_publisher.log}"
-
-HEAD_IP="$(container_ip "${HEAD_CONTAINER}")"
-
-echo "[head] start discovery publisher (supervised): ${HEAD_IP_FILE} (head_ip=${HEAD_IP})"
-
-# stop existing (best-effort)
-dexec "${HEAD_CONTAINER}" bash -lc "if test -f '${PID_PATH}'; then pid=\$(cat '${PID_PATH}'); if kill -0 \"\${pid}\" >/dev/null 2>&1; then kill \"\${pid}\" || true; fi; rm -f '${PID_PATH}'; fi"
-
-dexec "${HEAD_CONTAINER}" bash -lc "mkdir -p \"$(dirname "${LOG_PATH}")\" \"$(dirname "${HEAD_IP_FILE}")\""
-
-# Supervisor loop: restart publisher if it exits.
-docker exec -d "${HEAD_CONTAINER}" bash -lc "
-  nohup bash -lc '
-    export PYTHONPATH=/workspace/mvp/py
-    while true; do
-      python3 -m argus.ray.head_publisher \
-        --cluster-name \"${CLUSTER_NAME}\" \
-        --head-ip-file \"${HEAD_IP_FILE}\" \
-        --head-ip \"${HEAD_IP}\" \
-        --ttl-s \"${TTL_S}\" \
-        --refresh-s \"${REFRESH_S}\"
-      sleep 2
-    done
-  ' >>'${LOG_PATH}' 2>&1 & echo \$! >'${PID_PATH}'
-"
-
-echo "[head] publisher pid stored in ${PID_PATH} (container-local)"
-echo "[head] logs: ${LOG_PATH}"

+echo "[head] discovery publisher is supervised in-container; verify head record exists: ${HEAD_IP_FILE}"
+dexec "${HEAD_CONTAINER}" bash -lc "test -f '${HEAD_IP_FILE}' && python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[\"job_server_url\"])' '${HEAD_IP_FILE}' || true"
--- a/src/mvp/scripts/23_start_workers_stateless.sh
+++ b/src/mvp/scripts/23_start_workers_stateless.sh
@@ -5,48 +5,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=lib.sh
 source "${SCRIPT_DIR}/lib.sh"

-CLUSTER_NAME="${CLUSTER_NAME:-argus-ray}"
-HEAD_IP_FILE="${HEAD_IP_FILE:-${SHARED_ROOT}/ray/discovery/${CLUSTER_NAME}/head.json}"
-POLL_S="${POLL_S:-5}"
-WORKER_NODE_RESOURCE="${WORKER_NODE_RESOURCE:-100}"
-
-start_one() {
-  local worker="$1"
-  local ip
-  ip="$(container_ip "${worker}")"
-
-  local pid_path="/tmp/argus_worker_watchdog.pid"
-  local log_path="${SHARED_ROOT}/common/logs/argus_worker_watchdog.${worker}.log"
-
-  echo "[${worker}] start stateless watchdog (supervised): head_file=${HEAD_IP_FILE} node_ip=${ip}"
-
-  # stop existing watchdog (best-effort)
-  dexec "${worker}" bash -lc "if test -f '${pid_path}'; then pid=\$(cat '${pid_path}'); if kill -0 \"\${pid}\" >/dev/null 2>&1; then kill \"\${pid}\" || true; fi; rm -f '${pid_path}'; fi"
-
-  # stop any legacy ray process to avoid split-brain
-  dexec "${worker}" bash -lc "ray stop --force || true"
-  dexec "${worker}" bash -lc "mkdir -p \"$(dirname "${log_path}")\""
-
-  docker exec -d "${worker}" bash -lc "
-    nohup bash -lc '
-      export PYTHONPATH=/workspace/mvp/py
-      while true; do
-        python3 -m argus.ray.worker_watchdog \
-          --head-ip-file \"${HEAD_IP_FILE}\" \
-          --node-ip \"${ip}\" \
-          --resources-kv \"worker_node=${WORKER_NODE_RESOURCE}\" \
-          --poll-s \"${POLL_S}\"
-        sleep 2
-      done
-    ' >>'${log_path}' 2>&1 & echo \$! >'${pid_path}'
-  "
-
-  echo "[${worker}] watchdog pid stored in ${pid_path} (container-local)"
-  echo "[${worker}] logs: ${log_path}"
-}
-
-start_one "${WORKER0_CONTAINER}"
-start_one "${WORKER1_CONTAINER}"
+echo "[workers] restart worker containers (ray is supervised in-container)"
+docker restart "${WORKER0_CONTAINER}" >/dev/null
+docker restart "${WORKER1_CONTAINER}" >/dev/null

 echo "[head] ray status"
 dexec "${HEAD_CONTAINER}" bash -lc "ray status || true"