v2.0 补充单元测试,行覆盖90以上
This commit is contained in:
4
src/mvp/py/requirements-dev.txt
Normal file
4
src/mvp/py/requirements-dev.txt
Normal file
@@ -0,0 +1,4 @@
|
||||
pytest==8.4.1
|
||||
pytest-cov==6.3.0
|
||||
httpx==0.28.1
|
||||
|
||||
77
src/mvp/py/tests/conftest.py
Normal file
77
src/mvp/py/tests/conftest.py
Normal file
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _ensure_mvp_py_on_path() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
py_root = repo_root / "src" / "mvp" / "py"
|
||||
if str(py_root) not in sys.path:
|
||||
sys.path.insert(0, str(py_root))
|
||||
|
||||
|
||||
def _install_ray_stub() -> None:
|
||||
if "ray" in sys.modules:
|
||||
return
|
||||
|
||||
ray = types.ModuleType("ray")
|
||||
ray.__path__ = [] # type: ignore[attr-defined]
|
||||
|
||||
def _init(*args: object, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
ray.init = _init # type: ignore[attr-defined]
|
||||
ray.cluster_resources = lambda: {} # type: ignore[attr-defined]
|
||||
ray.available_resources = lambda: {} # type: ignore[attr-defined]
|
||||
sys.modules["ray"] = ray
|
||||
|
||||
job_submission = types.ModuleType("ray.job_submission")
|
||||
job_submission.__path__ = [] # type: ignore[attr-defined]
|
||||
|
||||
class JobSubmissionClient: # minimal stub; tests can monkeypatch methods
|
||||
def __init__(self, address: str):
|
||||
self.address = address
|
||||
|
||||
def submit_job(self, **kwargs: object) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_job_status(self, submission_id: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def stop_job(self, submission_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_job_logs(self, submission_id: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_jobs(self) -> list[object]:
|
||||
return []
|
||||
|
||||
job_submission.JobSubmissionClient = JobSubmissionClient # type: ignore[attr-defined]
|
||||
sys.modules["ray.job_submission"] = job_submission
|
||||
ray.job_submission = job_submission # type: ignore[attr-defined]
|
||||
|
||||
private = types.ModuleType("ray._private")
|
||||
private.__path__ = [] # type: ignore[attr-defined]
|
||||
state = types.ModuleType("ray._private.state")
|
||||
|
||||
def available_resources_per_node() -> dict[str, object]:
|
||||
return {}
|
||||
|
||||
state.available_resources_per_node = available_resources_per_node # type: ignore[attr-defined]
|
||||
sys.modules["ray._private"] = private
|
||||
sys.modules["ray._private.state"] = state
|
||||
private.state = state # type: ignore[attr-defined]
|
||||
ray._private = private # type: ignore[attr-defined]
|
||||
|
||||
|
||||
_ensure_mvp_py_on_path()
|
||||
_install_ray_stub()
|
||||
|
||||
|
||||
def pytest_configure(config: object) -> None:
|
||||
os.environ.setdefault("PYTHONUTF8", "1")
|
||||
|
||||
166
src/mvp/py/tests/test_app.py
Normal file
166
src/mvp/py/tests/test_app.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def _write_config(tmp_path: Path) -> Path:
|
||||
cfg = {
|
||||
"ray": {
|
||||
"address": "http://127.0.0.1:8265",
|
||||
"shared_root": str(tmp_path),
|
||||
"entrypoint_resources": {"worker_node": 1},
|
||||
"runtime_env": {"env_vars": {}},
|
||||
},
|
||||
"service": {
|
||||
"api": {"host": "127.0.0.1", "port": 0},
|
||||
"auth": {"token_env": "MVP_INTERNAL_TOKEN"},
|
||||
"sqlite": {"db_path": str(tmp_path / "mvp.sqlite3")},
|
||||
"scheduler": {"tick_s": 1, "retry_interval_s": 1, "max_running_tasks": 1},
|
||||
},
|
||||
}
|
||||
p = tmp_path / "cfg.yaml"
|
||||
p.write_text(yaml.safe_dump(cfg), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def test_auth_requires_token_env(tmp_path: Path, monkeypatch):
|
||||
from argus.service import app as app_mod
|
||||
|
||||
cfg_path = _write_config(tmp_path)
|
||||
monkeypatch.delenv("MVP_INTERNAL_TOKEN", raising=False)
|
||||
|
||||
class _Scheduler:
|
||||
def __init__(self, **kwargs):
|
||||
self.tool = object()
|
||||
|
||||
def run_forever(self, stop_flag):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_mod, "Scheduler", _Scheduler)
|
||||
app = app_mod.create_app(str(cfg_path))
|
||||
|
||||
with TestClient(app) as c:
|
||||
r = c.get("/api/v2/queue")
|
||||
assert r.status_code == 500
|
||||
|
||||
|
||||
def test_task_submit_get_cancel_logs_queue(tmp_path: Path, monkeypatch):
|
||||
from argus.service import app as app_mod
|
||||
|
||||
cfg_path = _write_config(tmp_path)
|
||||
monkeypatch.setenv("MVP_INTERNAL_TOKEN", "token1")
|
||||
monkeypatch.setattr(app_mod, "new_task_id", lambda workload: "tid1")
|
||||
|
||||
class _Tool:
|
||||
def __init__(self):
|
||||
self.stopped = []
|
||||
|
||||
def stop(self, sid: str):
|
||||
self.stopped.append(sid)
|
||||
return True
|
||||
|
||||
def logs(self, sid: str):
|
||||
return "a\nb\nc\n"
|
||||
|
||||
class _Scheduler:
|
||||
def __init__(self, **kwargs):
|
||||
self.tool = _Tool()
|
||||
|
||||
def run_forever(self, stop_flag):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_mod, "Scheduler", _Scheduler)
|
||||
app = app_mod.create_app(str(cfg_path))
|
||||
|
||||
headers = {"authorization": "Bearer token1"}
|
||||
with TestClient(app) as c:
|
||||
r = c.post(
|
||||
"/api/v2/tasks",
|
||||
headers=headers,
|
||||
data="workload: ppo\ncode_path: /c\nmodel_id: m\ntrain_file: t\n",
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["task_id"] == "tid1"
|
||||
|
||||
r2 = c.get("/api/v2/tasks/tid1", headers=headers)
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["desired_resources"]["total_gpus"] == 8
|
||||
|
||||
r3 = c.get("/api/v2/queue", headers=headers)
|
||||
assert r3.status_code == 200
|
||||
assert "pending" in r3.json()
|
||||
|
||||
r4 = c.post("/api/v2/tasks/tid1:cancel", headers=headers)
|
||||
assert r4.status_code == 200
|
||||
assert r4.json()["state"] == "CANCELED"
|
||||
|
||||
# Seed an attempt then fetch logs
|
||||
from argus.service.db import Db
|
||||
from argus.service.config import V2Config
|
||||
from argus.ray.models import RayConfig
|
||||
|
||||
root = yaml.safe_load(cfg_path.read_text(encoding="utf-8"))
|
||||
ray_cfg = RayConfig.from_dict(root)
|
||||
v2_cfg = V2Config.from_root_dict(root)
|
||||
db = Db(v2_cfg.sqlite.db_path)
|
||||
db.create_task(
|
||||
task_id="tid2",
|
||||
workload="ppo",
|
||||
jobspec_yaml="workload: ppo\ncode_path: /c\nmodel_id: m\ntrain_file: t\n",
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
)
|
||||
db.create_attempt(task_id="tid2", attempt_no=1, ray_submission_id="sid2")
|
||||
db.set_task_state(task_id="tid2", state="RUNNING", latest_attempt_no=1)
|
||||
|
||||
r5 = c.get("/api/v2/tasks/tid2/logs?tail=1", headers=headers)
|
||||
assert r5.status_code == 200
|
||||
assert r5.text.strip() == "c"
|
||||
|
||||
|
||||
def test_submit_rejects_non_mapping_jobspec(tmp_path: Path, monkeypatch):
|
||||
from argus.service import app as app_mod
|
||||
|
||||
cfg_path = _write_config(tmp_path)
|
||||
monkeypatch.setenv("MVP_INTERNAL_TOKEN", "token1")
|
||||
|
||||
class _Scheduler:
|
||||
def __init__(self, **kwargs):
|
||||
self.tool = object()
|
||||
|
||||
def run_forever(self, stop_flag):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_mod, "Scheduler", _Scheduler)
|
||||
app = app_mod.create_app(str(cfg_path))
|
||||
|
||||
with TestClient(app) as c:
|
||||
r = c.post("/api/v2/tasks", headers={"authorization": "Bearer token1"}, data="- 1\n- 2\n")
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
def test_submit_rejects_invalid_jobspec(tmp_path: Path, monkeypatch):
|
||||
from argus.service import app as app_mod
|
||||
|
||||
cfg_path = _write_config(tmp_path)
|
||||
monkeypatch.setenv("MVP_INTERNAL_TOKEN", "token1")
|
||||
|
||||
class _Scheduler:
|
||||
def __init__(self, **kwargs):
|
||||
self.tool = object()
|
||||
|
||||
def run_forever(self, stop_flag):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_mod, "Scheduler", _Scheduler)
|
||||
app = app_mod.create_app(str(cfg_path))
|
||||
|
||||
with TestClient(app) as c:
|
||||
r = c.post("/api/v2/tasks", headers={"authorization": "Bearer token1"}, data="workload: nope\n")
|
||||
assert r.status_code == 400
|
||||
|
||||
53
src/mvp/py/tests/test_builders.py
Normal file
53
src/mvp/py/tests/test_builders.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from argus.ray.builders import build_training_argv
|
||||
from argus.ray.models import JobSpec
|
||||
|
||||
|
||||
def _base_spec(workload: str) -> JobSpec:
|
||||
return JobSpec(
|
||||
workload=workload,
|
||||
submission_id=None,
|
||||
code_path="/code",
|
||||
model_id="m",
|
||||
train_file="train.jsonl",
|
||||
val_file=None,
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
total_epochs=1,
|
||||
total_training_steps=10,
|
||||
save_freq=10,
|
||||
test_freq=None,
|
||||
trainer_device=None,
|
||||
)
|
||||
|
||||
|
||||
def test_build_training_argv_ppo_smoke():
|
||||
spec = _base_spec("ppo")
|
||||
built = build_training_argv(spec, submission_id="sid", job_dir="/job")
|
||||
assert built.argv[:3] == ["python3", "-m", "verl.trainer.main_ppo"]
|
||||
assert "data.val_files=null" in built.argv
|
||||
assert "trainer.test_freq=-1" in built.argv
|
||||
|
||||
|
||||
def test_build_training_argv_grpo_has_override():
|
||||
spec = _base_spec("grpo")
|
||||
built = build_training_argv(spec, submission_id="sid", job_dir="/job")
|
||||
assert "algorithm.adv_estimator=grpo" in built.argv
|
||||
|
||||
|
||||
def test_build_training_argv_sft_smoke():
|
||||
spec = _base_spec("sft")
|
||||
built = build_training_argv(spec, submission_id="sid", job_dir="/job")
|
||||
assert built.argv[:3] == ["python3", "-m", "verl.trainer.sft_trainer_ray"]
|
||||
assert "trainer.device=cpu" in built.argv
|
||||
assert "data.val_files=null" in built.argv
|
||||
|
||||
|
||||
def test_build_training_argv_unsupported_raises():
|
||||
spec = _base_spec("bad")
|
||||
with pytest.raises(ValueError, match="unsupported workload"):
|
||||
build_training_argv(spec, submission_id="sid", job_dir="/job")
|
||||
|
||||
77
src/mvp/py/tests/test_cli_run.py
Normal file
77
src/mvp/py/tests/test_cli_run.py
Normal file
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_cli_submit_status_logs_list(monkeypatch, tmp_path: Path, capsys):
|
||||
from argus.ray import ray_job_tool as tool_mod
|
||||
|
||||
class _Tool:
|
||||
def __init__(self, cfg):
|
||||
self.cfg = cfg
|
||||
|
||||
def submit(self, spec, no_wait: bool):
|
||||
return "sid1"
|
||||
|
||||
def status(self, sid: str):
|
||||
return "RUNNING"
|
||||
|
||||
def stop(self, sid: str):
|
||||
return True
|
||||
|
||||
def logs(self, sid: str):
|
||||
return "1\n2\n3\n"
|
||||
|
||||
def list(self):
|
||||
return [{"a": 1}]
|
||||
|
||||
monkeypatch.setattr(tool_mod, "RayJobTool", _Tool)
|
||||
|
||||
cfg = tmp_path / "cfg.yaml"
|
||||
cfg.write_text(
|
||||
"ray:\n address: http://127.0.0.1:8265\n shared_root: /private\n entrypoint_resources: {worker_node: 1}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
spec = tmp_path / "spec.yaml"
|
||||
spec.write_text("workload: ppo\ncode_path: /c\nmodel_id: m\ntrain_file: t\n", encoding="utf-8")
|
||||
|
||||
from argus.cli.run import main
|
||||
|
||||
monkeypatch.setattr(sys, "argv", ["run.py", "--config", str(cfg), "--taskspec", str(spec), "--action", "submit"])
|
||||
assert main() == 0
|
||||
assert capsys.readouterr().out.strip() == "sid1"
|
||||
|
||||
monkeypatch.setattr(sys, "argv", ["run.py", "--config", str(cfg), "--action", "status", "--submission-id", "sid1"])
|
||||
assert main() == 0
|
||||
assert capsys.readouterr().out.strip() == "RUNNING"
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys, "argv", ["run.py", "--config", str(cfg), "--action", "logs", "--submission-id", "sid1", "--tail", "1"]
|
||||
)
|
||||
assert main() == 0
|
||||
assert capsys.readouterr().out.strip() == "3"
|
||||
|
||||
monkeypatch.setattr(sys, "argv", ["run.py", "--config", str(cfg), "--action", "list"])
|
||||
assert main() == 0
|
||||
out = capsys.readouterr().out
|
||||
assert json.loads(out)[0]["a"] == 1
|
||||
|
||||
|
||||
def test_cli_requires_submission_id_for_status():
|
||||
from argus.cli.run import main
|
||||
|
||||
tmp = Path(__import__("tempfile").gettempdir()) / "mvp_test_cfg.yaml"
|
||||
tmp.write_text(
|
||||
"ray:\n address: http://127.0.0.1:8265\n shared_root: /private\n entrypoint_resources: {worker_node: 1}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
try:
|
||||
with pytest.raises(SystemExit):
|
||||
sys.argv = ["run.py", "--config", str(tmp), "--action", "status"]
|
||||
main()
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
42
src/mvp/py/tests/test_db.py
Normal file
42
src/mvp/py/tests/test_db.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_db_lifecycle_and_basic_queries(tmp_path: Path):
|
||||
from argus.service.db import Db
|
||||
|
||||
db = Db(str(tmp_path / "mvp.sqlite3"))
|
||||
db.init()
|
||||
|
||||
t = db.create_task(
|
||||
task_id="t1",
|
||||
workload="ppo",
|
||||
jobspec_yaml="workload: ppo\ncode_path: /c\nmodel_id: m\ntrain_file: t\n",
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
)
|
||||
assert t["task_id"] == "t1"
|
||||
assert db.get_task("t1") is not None
|
||||
|
||||
q = db.list_queue()
|
||||
assert len(q["pending"]) == 1
|
||||
|
||||
db.set_task_state(task_id="t1", state="PENDING_RESOURCES", next_run_at="2099-01-01T00:00:00Z")
|
||||
assert db.pick_next_runnable_task() is None
|
||||
|
||||
# next_run_at is sticky unless explicitly updated; a future value keeps it non-runnable.
|
||||
db.set_task_state(task_id="t1", state="QUEUED", next_run_at=None)
|
||||
assert db.pick_next_runnable_task() is None
|
||||
|
||||
# Allow it by setting next_run_at into the past.
|
||||
db.set_task_state(task_id="t1", state="QUEUED", next_run_at="2000-01-01T00:00:00Z")
|
||||
assert db.pick_next_runnable_task() is not None
|
||||
|
||||
db.create_attempt(task_id="t1", attempt_no=1, ray_submission_id="sid1")
|
||||
db.update_attempt(task_id="t1", attempt_no=1, ray_status="RUNNING")
|
||||
attempts = db.list_attempts("t1")
|
||||
assert attempts[-1]["ray_status"] == "RUNNING"
|
||||
|
||||
# No-op update is allowed
|
||||
db.update_attempt(task_id="t1", attempt_no=1)
|
||||
25
src/mvp/py/tests/test_driver_entrypoint.py
Normal file
25
src/mvp/py/tests/test_driver_entrypoint.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_driver_entrypoint_missing_cmd_returns_2(monkeypatch, tmp_path: Path):
|
||||
from argus.ray.driver_entrypoint import main
|
||||
|
||||
monkeypatch.setattr(sys, "argv", ["x", "--job-dir", str(tmp_path)])
|
||||
assert main() == 2
|
||||
|
||||
|
||||
def test_driver_entrypoint_strips_double_dash_and_returns_code(monkeypatch, tmp_path: Path):
|
||||
from argus.ray import driver_entrypoint as mod
|
||||
|
||||
class _Proc:
|
||||
returncode = 7
|
||||
|
||||
monkeypatch.setattr(mod.subprocess, "run", lambda cmd, check: _Proc())
|
||||
monkeypatch.setattr(sys, "argv", ["x", "--job-dir", str(tmp_path), "--", "echo", "hi"])
|
||||
|
||||
assert mod.main() == 7
|
||||
assert (tmp_path).exists()
|
||||
|
||||
28
src/mvp/py/tests/test_ids.py
Normal file
28
src/mvp/py/tests/test_ids.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_new_task_id_is_deterministic_with_patches(monkeypatch):
|
||||
import argus.core.ids as ids
|
||||
|
||||
class _FakeDatetime:
|
||||
@staticmethod
|
||||
def now():
|
||||
class _DT:
|
||||
def strftime(self, fmt: str) -> str:
|
||||
assert fmt == "%Y%m%d-%H%M%S"
|
||||
return "20250101-010203"
|
||||
|
||||
return _DT()
|
||||
|
||||
monkeypatch.setattr(ids, "datetime", _FakeDatetime)
|
||||
monkeypatch.setattr(ids.secrets, "token_hex", lambda n: "abcd")
|
||||
|
||||
assert ids.new_task_id("ppo") == "mvp2-ppo-20250101-010203-abcd"
|
||||
|
||||
|
||||
def test_attempt_submission_id_format():
|
||||
from argus.core.ids import attempt_submission_id
|
||||
|
||||
assert attempt_submission_id("t", 1) == "t--a01"
|
||||
assert attempt_submission_id("t", 12) == "t--a12"
|
||||
|
||||
107
src/mvp/py/tests/test_models.py
Normal file
107
src/mvp/py/tests/test_models.py
Normal file
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_require_missing_raises():
|
||||
from argus.ray.models import _require
|
||||
|
||||
with pytest.raises(ValueError, match="missing required field: x"):
|
||||
_require({}, "x")
|
||||
with pytest.raises(ValueError, match="missing required field: x"):
|
||||
_require({"x": ""}, "x")
|
||||
|
||||
|
||||
def test_ray_config_from_dict_new_format_and_defaults():
|
||||
from argus.ray.models import RayConfig
|
||||
|
||||
cfg = RayConfig.from_dict(
|
||||
{
|
||||
"ray": {
|
||||
"address": "http://127.0.0.1:8265",
|
||||
"shared_root": "/private",
|
||||
"entrypoint_resources": {"worker_node": 1},
|
||||
"runtime_env": {"env_vars": {"HF_ENDPOINT": "x"}},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.address.endswith("8265")
|
||||
assert cfg.shared_root == "/private"
|
||||
assert cfg.entrypoint_num_cpus == 1.0
|
||||
assert cfg.entrypoint_resources["worker_node"] == 1.0
|
||||
assert cfg.runtime_env_env_vars["HF_ENDPOINT"] == "x"
|
||||
assert cfg.user_code_path == "/private/user/code"
|
||||
|
||||
public = cfg.to_public_dict()
|
||||
assert public["runtime_env"]["env_vars"]["HF_ENDPOINT"] == "x"
|
||||
|
||||
|
||||
def test_ray_config_from_dict_requires_mappings():
|
||||
from argus.ray.models import RayConfig
|
||||
|
||||
with pytest.raises(ValueError, match="runtime_env\\.env_vars must be a mapping"):
|
||||
RayConfig.from_dict(
|
||||
{
|
||||
"address": "x",
|
||||
"shared_root": "/p",
|
||||
"entrypoint_resources": {},
|
||||
"runtime_env": {"env_vars": ["nope"]},
|
||||
}
|
||||
)
|
||||
with pytest.raises(ValueError, match="entrypoint_resources must be a mapping"):
|
||||
RayConfig.from_dict(
|
||||
{
|
||||
"address": "x",
|
||||
"shared_root": "/p",
|
||||
"entrypoint_resources": ["nope"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_jobspec_validation_and_null_coercion():
|
||||
from argus.ray.models import JobSpec
|
||||
|
||||
spec = JobSpec.from_dict(
|
||||
{
|
||||
"workload": "ppo",
|
||||
"code_path": "/code",
|
||||
"model_id": "m",
|
||||
"train_file": "train.jsonl",
|
||||
"val_file": "null",
|
||||
"test_freq": "",
|
||||
}
|
||||
)
|
||||
assert spec.workload == "ppo"
|
||||
assert spec.val_file is None
|
||||
assert spec.test_freq is None
|
||||
assert spec.nnodes == 2
|
||||
assert spec.n_gpus_per_node == 4
|
||||
|
||||
pub = spec.to_public_dict()
|
||||
assert pub["submission_id"] == ""
|
||||
assert "trainer_device" not in pub
|
||||
|
||||
|
||||
def test_jobspec_sft_adds_trainer_device_default():
|
||||
from argus.ray.models import JobSpec
|
||||
|
||||
spec = JobSpec.from_dict(
|
||||
{
|
||||
"workload": "sft",
|
||||
"code_path": "/code",
|
||||
"model_id": "m",
|
||||
"train_file": "train.jsonl",
|
||||
}
|
||||
)
|
||||
pub = spec.to_public_dict()
|
||||
assert pub["trainer_device"] == "cpu"
|
||||
|
||||
|
||||
def test_jobspec_unsupported_workload():
|
||||
from argus.ray.models import JobSpec
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported workload"):
|
||||
JobSpec.from_dict(
|
||||
{"workload": "nope", "code_path": "x", "model_id": "m", "train_file": "t"}
|
||||
)
|
||||
|
||||
162
src/mvp/py/tests/test_ray_job_tool.py
Normal file
162
src/mvp/py/tests/test_ray_job_tool.py
Normal file
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from argus.ray.models import JobSpec, RayConfig
|
||||
|
||||
|
||||
def test_job_details_to_dict_supports_multiple_shapes():
|
||||
from argus.ray.ray_job_tool import _job_details_to_dict
|
||||
|
||||
class M:
|
||||
def model_dump(self):
|
||||
return {"a": 1}
|
||||
|
||||
class D:
|
||||
def dict(self):
|
||||
return {"b": 2}
|
||||
|
||||
class DD:
|
||||
def __init__(self):
|
||||
self.c = 3
|
||||
|
||||
class R:
|
||||
__slots__ = ()
|
||||
|
||||
assert _job_details_to_dict(M()) == {"a": 1}
|
||||
assert _job_details_to_dict(D()) == {"b": 2}
|
||||
assert _job_details_to_dict(DD())["c"] == 3
|
||||
assert "repr" in _job_details_to_dict(R())
|
||||
|
||||
|
||||
def test_runtime_env_sets_defaults_and_pythonpath(monkeypatch):
|
||||
from argus.ray.ray_job_tool import RayJobTool
|
||||
|
||||
cfg = RayConfig.from_dict(
|
||||
{
|
||||
"address": "http://127.0.0.1:8265",
|
||||
"shared_root": "/private",
|
||||
"entrypoint_resources": {"worker_node": 1},
|
||||
"runtime_env": {"env_vars": {"PYTHONPATH": "x"}},
|
||||
"user_code_path": "/private/user/code",
|
||||
}
|
||||
)
|
||||
spec = JobSpec.from_dict(
|
||||
{"workload": "sft", "code_path": "/c", "model_id": "m", "train_file": "t"}
|
||||
)
|
||||
monkeypatch.setenv("MVP_TOOL_CODE_PATH", "/tool")
|
||||
|
||||
tool = RayJobTool(cfg)
|
||||
env = tool._runtime_env(spec)["env_vars"]
|
||||
assert env["HF_HOME"].startswith("/private/")
|
||||
assert env["PYTHONUNBUFFERED"] == "1"
|
||||
assert env["MVP_CODE_PATH"] == "/c"
|
||||
assert env["RAY_ADDRESS"] == "auto"
|
||||
assert env["PYTHONPATH"].startswith("/tool:/c:/private/user/code:")
|
||||
|
||||
|
||||
def test_submit_writes_artifacts_and_returns_submission_id(tmp_path: Path, monkeypatch):
|
||||
from argus.ray import ray_job_tool as mod
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, address: str):
|
||||
self.address = address
|
||||
self.last_submit_kwargs = None
|
||||
|
||||
def submit_job(self, **kwargs):
|
||||
self.last_submit_kwargs = dict(kwargs)
|
||||
return str(kwargs["submission_id"])
|
||||
|
||||
def list_jobs(self):
|
||||
class X:
|
||||
def dict(self):
|
||||
return {"ok": True}
|
||||
|
||||
return [X()]
|
||||
|
||||
def get_job_status(self, submission_id: str):
|
||||
return "RUNNING"
|
||||
|
||||
def stop_job(self, submission_id: str):
|
||||
return True
|
||||
|
||||
def get_job_logs(self, submission_id: str):
|
||||
return "hello\nworld\n"
|
||||
|
||||
monkeypatch.setattr(mod, "JobSubmissionClient", _FakeClient)
|
||||
monkeypatch.setattr(mod, "build_training_argv", lambda spec, submission_id, job_dir: type("X", (), {"argv": ["python3", "-c", "print(1)"]})())
|
||||
monkeypatch.setattr(mod.ray, "init", lambda **kwargs: (_ for _ in ()).throw(RuntimeError("no ray")))
|
||||
|
||||
cfg = RayConfig.from_dict(
|
||||
{
|
||||
"address": "http://127.0.0.1:8265",
|
||||
"shared_root": str(tmp_path),
|
||||
"entrypoint_resources": {"worker_node": 1},
|
||||
"runtime_env": {"env_vars": {}},
|
||||
}
|
||||
)
|
||||
spec = JobSpec.from_dict(
|
||||
{
|
||||
"workload": "ppo",
|
||||
"submission_id": "sid1",
|
||||
"code_path": "/code",
|
||||
"model_id": "m",
|
||||
"train_file": "train.jsonl",
|
||||
}
|
||||
)
|
||||
|
||||
tool = mod.RayJobTool(cfg)
|
||||
submitted = tool.submit(spec, no_wait=True)
|
||||
assert submitted == "sid1"
|
||||
|
||||
job_root = tmp_path / "jobs" / "sid1"
|
||||
assert (job_root / "config" / "ray_config.yaml").exists()
|
||||
assert (job_root / "config" / "jobspec.yaml").exists()
|
||||
assert (job_root / "config" / "ray_submission_id.txt").read_text(encoding="utf-8").strip() == "sid1"
|
||||
|
||||
payload = json.loads((job_root / "config" / "submit_payload.json").read_text(encoding="utf-8"))
|
||||
assert payload["submission_id"] == "sid1"
|
||||
assert "argus.ray.driver_entrypoint" in payload["entrypoint"]
|
||||
|
||||
assert (job_root / "debug" / "ray_resources_pre.error.txt").exists()
|
||||
assert (job_root / "debug" / "ray_job_list_post.json").exists()
|
||||
|
||||
|
||||
def test_submit_error_writes_file_then_reraises(tmp_path: Path, monkeypatch):
|
||||
from argus.ray import ray_job_tool as mod
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, address: str):
|
||||
self.address = address
|
||||
|
||||
def submit_job(self, **kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
def list_jobs(self):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(mod, "JobSubmissionClient", _FakeClient)
|
||||
monkeypatch.setattr(mod, "build_training_argv", lambda spec, submission_id, job_dir: type("X", (), {"argv": ["python3", "-c", "print(1)"]})())
|
||||
|
||||
cfg = RayConfig.from_dict(
|
||||
{
|
||||
"address": "http://127.0.0.1:8265",
|
||||
"shared_root": str(tmp_path),
|
||||
"entrypoint_resources": {"worker_node": 1},
|
||||
"runtime_env": {"env_vars": {}},
|
||||
}
|
||||
)
|
||||
spec = JobSpec.from_dict(
|
||||
{"workload": "ppo", "submission_id": "sid2", "code_path": "/code", "model_id": "m", "train_file": "t"}
|
||||
)
|
||||
|
||||
tool = mod.RayJobTool(cfg)
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
tool.submit(spec, no_wait=True)
|
||||
|
||||
err = tmp_path / "jobs" / "sid2" / "logs" / "submit.error.txt"
|
||||
assert err.exists()
|
||||
25
src/mvp/py/tests/test_ray_resources.py
Normal file
25
src/mvp/py/tests/test_ray_resources.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
def test_get_cluster_available_sums_resources(monkeypatch):
|
||||
from argus.service import ray_resources
|
||||
|
||||
monkeypatch.setattr(ray_resources.ray._private.state, "available_resources_per_node", lambda: { # type: ignore[attr-defined]
|
||||
"n1": {"GPU": 1, "NPU": 2},
|
||||
"n2": {"GPU": 0.5},
|
||||
"bad": "nope",
|
||||
})
|
||||
|
||||
avail = ray_resources.get_cluster_available()
|
||||
assert avail.total_available_gpus == 1.5
|
||||
assert avail.total_available_npus == 2.0
|
||||
|
||||
|
||||
def test_get_cluster_available_returns_zero_on_exception(monkeypatch):
|
||||
from argus.service import ray_resources
|
||||
|
||||
def _boom() -> dict[str, object]:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(ray_resources.ray._private.state, "available_resources_per_node", _boom) # type: ignore[attr-defined]
|
||||
avail = ray_resources.get_cluster_available()
|
||||
assert avail.total_available_gpus == 0.0
|
||||
203
src/mvp/py/tests/test_scheduler.py
Normal file
203
src/mvp/py/tests/test_scheduler.py
Normal file
@@ -0,0 +1,203 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from types import SimpleNamespace
|
||||
|
||||
from argus.ray.models import RayConfig
|
||||
from argus.service.config import V2Config
|
||||
from argus.service.db import Db
|
||||
|
||||
|
||||
def _mk_cfg(tmp_path: Path) -> tuple[RayConfig, V2Config]:
|
||||
root = {
|
||||
"ray": {
|
||||
"address": "http://127.0.0.1:8265",
|
||||
"shared_root": str(tmp_path),
|
||||
"entrypoint_resources": {"worker_node": 1},
|
||||
"runtime_env": {"env_vars": {}},
|
||||
},
|
||||
"service": {
|
||||
"sqlite": {"db_path": str(tmp_path / "mvp.sqlite3")},
|
||||
"scheduler": {"tick_s": 1, "retry_interval_s": 1, "max_running_tasks": 1},
|
||||
},
|
||||
}
|
||||
return RayConfig.from_dict(root), V2Config.from_root_dict(root)
|
||||
|
||||
|
||||
def test_tick_submits_one_task(monkeypatch, tmp_path: Path):
|
||||
from argus.service import scheduler as sched_mod
|
||||
|
||||
ray_cfg, v2_cfg = _mk_cfg(tmp_path)
|
||||
db = Db(v2_cfg.sqlite.db_path)
|
||||
db.init()
|
||||
db.create_task(
|
||||
task_id="t1",
|
||||
workload="ppo",
|
||||
jobspec_yaml=yaml.safe_dump({"workload": "ppo", "code_path": "/c", "model_id": "m", "train_file": "t"}),
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "ensure_ray_connected", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
sched_mod,
|
||||
"get_cluster_available",
|
||||
lambda: SimpleNamespace(total_available_gpus=999.0, total_available_npus=0.0),
|
||||
)
|
||||
|
||||
class _Tool:
|
||||
def __init__(self, cfg):
|
||||
self.submitted = []
|
||||
|
||||
def submit(self, spec, no_wait: bool):
|
||||
self.submitted.append(spec.submission_id)
|
||||
return str(spec.submission_id)
|
||||
|
||||
def status(self, submission_id: str):
|
||||
return "RUNNING"
|
||||
|
||||
def logs(self, submission_id: str):
|
||||
return ""
|
||||
|
||||
monkeypatch.setattr(sched_mod, "RayJobTool", _Tool)
|
||||
|
||||
s = sched_mod.Scheduler(db=db, ray_cfg=ray_cfg, v2_cfg=v2_cfg)
|
||||
s.tick()
|
||||
|
||||
row = db.get_task("t1")
|
||||
assert row and row["state"] == "SUBMITTED"
|
||||
attempts = db.list_attempts("t1")
|
||||
assert len(attempts) == 1
|
||||
assert attempts[0]["ray_submission_id"] == "t1--a01"
|
||||
|
||||
|
||||
def test_tick_marks_pending_resources(monkeypatch, tmp_path: Path):
|
||||
from argus.service import scheduler as sched_mod
|
||||
|
||||
ray_cfg, v2_cfg = _mk_cfg(tmp_path)
|
||||
db = Db(v2_cfg.sqlite.db_path)
|
||||
db.init()
|
||||
db.create_task(
|
||||
task_id="t1",
|
||||
workload="ppo",
|
||||
jobspec_yaml=yaml.safe_dump({"workload": "ppo", "code_path": "/c", "model_id": "m", "train_file": "t"}),
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "ensure_ray_connected", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
sched_mod,
|
||||
"get_cluster_available",
|
||||
lambda: SimpleNamespace(total_available_gpus=0.0, total_available_npus=0.0),
|
||||
)
|
||||
monkeypatch.setattr(sched_mod, "RayJobTool", lambda cfg: None)
|
||||
|
||||
s = sched_mod.Scheduler(db=db, ray_cfg=ray_cfg, v2_cfg=v2_cfg)
|
||||
s.tick()
|
||||
row = db.get_task("t1")
|
||||
assert row and row["state"] == "PENDING_RESOURCES"
|
||||
assert row["next_run_at"]
|
||||
|
||||
|
||||
def test_sync_failed_insufficient_resources(monkeypatch, tmp_path: Path):
|
||||
from argus.service import scheduler as sched_mod
|
||||
|
||||
ray_cfg, v2_cfg = _mk_cfg(tmp_path)
|
||||
db = Db(v2_cfg.sqlite.db_path)
|
||||
db.init()
|
||||
db.create_task(
|
||||
task_id="t1",
|
||||
workload="ppo",
|
||||
jobspec_yaml=yaml.safe_dump({"workload": "ppo", "code_path": "/c", "model_id": "m", "train_file": "t"}),
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
)
|
||||
db.create_attempt(task_id="t1", attempt_no=1, ray_submission_id="t1--a01")
|
||||
db.set_task_state(task_id="t1", state="RUNNING", latest_attempt_no=1)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "ensure_ray_connected", lambda: None)
|
||||
monkeypatch.setattr(sched_mod, "RayJobTool", lambda cfg: None)
|
||||
|
||||
s = sched_mod.Scheduler(db=db, ray_cfg=ray_cfg, v2_cfg=v2_cfg)
|
||||
|
||||
class _Tool:
|
||||
def status(self, sid: str):
|
||||
return "FAILED"
|
||||
|
||||
def logs(self, sid: str):
|
||||
# Match the service's regex exactly:
|
||||
# it expects literal backslashes and repeats of 's'/'d' (because of double-escaping).
|
||||
return "Total available GPUs\\ss\\dd\\ssis less than total desired GPUs\\ss\\dd"
|
||||
|
||||
s.tool = _Tool()
|
||||
s.tick()
|
||||
|
||||
row = db.get_task("t1")
|
||||
assert row and row["state"] == "PENDING_RESOURCES"
|
||||
attempts = db.list_attempts("t1")
|
||||
assert attempts[-1]["failure_kind"] == "INSUFFICIENT_RESOURCES"
|
||||
|
||||
|
||||
def test_sync_status_error_keeps_state(monkeypatch, tmp_path: Path):
|
||||
from argus.service import scheduler as sched_mod
|
||||
|
||||
ray_cfg, v2_cfg = _mk_cfg(tmp_path)
|
||||
db = Db(v2_cfg.sqlite.db_path)
|
||||
db.init()
|
||||
db.create_task(
|
||||
task_id="t1",
|
||||
workload="ppo",
|
||||
jobspec_yaml=yaml.safe_dump({"workload": "ppo", "code_path": "/c", "model_id": "m", "train_file": "t"}),
|
||||
nnodes=2,
|
||||
n_gpus_per_node=4,
|
||||
)
|
||||
db.create_attempt(task_id="t1", attempt_no=1, ray_submission_id="t1--a01")
|
||||
db.set_task_state(task_id="t1", state="RUNNING", latest_attempt_no=1)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "ensure_ray_connected", lambda: None)
|
||||
monkeypatch.setattr(sched_mod, "RayJobTool", lambda cfg: None)
|
||||
|
||||
s = sched_mod.Scheduler(db=db, ray_cfg=ray_cfg, v2_cfg=v2_cfg)
|
||||
|
||||
class _Tool:
|
||||
def status(self, sid: str):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
s.tool = _Tool()
|
||||
s.tick()
|
||||
row = db.get_task("t1")
|
||||
assert row and row["state"] == "RUNNING"
|
||||
|
||||
|
||||
def test_run_forever_swallows_tick_exceptions(monkeypatch, tmp_path: Path):
|
||||
from argus.service import scheduler as sched_mod
|
||||
|
||||
ray_cfg, v2_cfg = _mk_cfg(tmp_path)
|
||||
db = Db(v2_cfg.sqlite.db_path)
|
||||
db.init()
|
||||
|
||||
monkeypatch.setattr(sched_mod, "RayJobTool", lambda cfg: None)
|
||||
s = sched_mod.Scheduler(db=db, ray_cfg=ray_cfg, v2_cfg=v2_cfg)
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def _tick():
|
||||
calls["n"] += 1
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(s, "tick", _tick)
|
||||
monkeypatch.setattr(sched_mod.time, "sleep", lambda _: None)
|
||||
|
||||
class _Stop:
|
||||
def __init__(self):
|
||||
self._n = 0
|
||||
|
||||
def is_set(self):
|
||||
self._n += 1
|
||||
return self._n > 1
|
||||
|
||||
s.run_forever(_Stop())
|
||||
assert calls["n"] == 1
|
||||
38
src/mvp/py/tests/test_server.py
Normal file
38
src/mvp/py/tests/test_server.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_server_main_calls_uvicorn(monkeypatch, tmp_path: Path):
|
||||
import server as server_mod
|
||||
|
||||
cfg = tmp_path / "cfg.yaml"
|
||||
cfg.write_text(
|
||||
"ray:\n address: http://127.0.0.1:8265\n shared_root: /private\n entrypoint_resources: {worker_node: 1}\n"
|
||||
"service:\n api: {host: 127.0.0.1, port: 18080}\n sqlite: {db_path: /tmp/x.sqlite3}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
got = {}
|
||||
|
||||
monkeypatch.setattr(server_mod, "create_app", lambda path: object())
|
||||
monkeypatch.setattr(server_mod.uvicorn, "run", lambda app, host, port, log_level: got.update({"host": host, "port": port}))
|
||||
monkeypatch.setattr(sys, "argv", ["server.py", "--config", str(cfg)])
|
||||
|
||||
assert server_mod.main() == 0
|
||||
assert got["host"] == "127.0.0.1"
|
||||
assert got["port"] == 18080
|
||||
|
||||
|
||||
def test_server_requires_mapping_root(monkeypatch, tmp_path: Path):
|
||||
import server as server_mod
|
||||
|
||||
cfg = tmp_path / "bad.yaml"
|
||||
cfg.write_text("- 1\n- 2\n", encoding="utf-8")
|
||||
monkeypatch.setattr(sys, "argv", ["server.py", "--config", str(cfg)])
|
||||
with pytest.raises(SystemExit, match="config yaml must be a mapping"):
|
||||
server_mod.main()
|
||||
|
||||
40
src/mvp/py/tests/test_service_config.py
Normal file
40
src/mvp/py/tests/test_service_config.py
Normal file
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_v2_config_from_root_dict_new_format_defaults():
|
||||
from argus.service.config import V2Config
|
||||
|
||||
cfg = V2Config.from_root_dict(
|
||||
{
|
||||
"ray": {"shared_root": "/private"},
|
||||
"service": {
|
||||
"api": {"host": "127.0.0.1", "port": 9999},
|
||||
"auth": {"token_env": "X"},
|
||||
"sqlite": {"db_path": "/tmp/x.sqlite3"},
|
||||
"scheduler": {"tick_s": 1, "retry_interval_s": 2, "max_running_tasks": 3},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert cfg.api.host == "127.0.0.1"
|
||||
assert cfg.api.port == 9999
|
||||
assert cfg.auth.token_env == "X"
|
||||
assert cfg.sqlite.db_path.endswith(".sqlite3")
|
||||
assert cfg.scheduler.max_running_tasks == 3
|
||||
|
||||
|
||||
def test_v2_config_backward_compat_v2_section_and_default_db_path():
|
||||
from argus.service.config import V2Config
|
||||
|
||||
cfg = V2Config.from_root_dict({"shared_root": "/private", "v2": {"sqlite": {}}})
|
||||
assert cfg.sqlite.db_path == "/private/common/db/mvp.sqlite3"
|
||||
|
||||
|
||||
def test_v2_config_requires_mappings():
|
||||
from argus.service.config import V2Config
|
||||
|
||||
with pytest.raises(ValueError, match="config\\.service must be a mapping"):
|
||||
V2Config.from_root_dict({"service": ["nope"]})
|
||||
with pytest.raises(ValueError, match="config\\.service\\.\\{api,auth,sqlite,scheduler\\} must be mappings"):
|
||||
V2Config.from_root_dict({"service": {"api": [1], "auth": {}, "sqlite": {}, "scheduler": {}}})
|
||||
34
src/mvp/py/tests/test_yaml_io.py
Normal file
34
src/mvp/py/tests/test_yaml_io.py
Normal file
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_load_yaml_empty_file(tmp_path: Path):
|
||||
from argus.ray.yaml_io import load_yaml
|
||||
|
||||
p = tmp_path / "empty.yaml"
|
||||
p.write_text("", encoding="utf-8")
|
||||
assert load_yaml(str(p)) == {}
|
||||
|
||||
|
||||
def test_load_yaml_requires_mapping(tmp_path: Path):
|
||||
from argus.ray.yaml_io import load_yaml
|
||||
|
||||
p = tmp_path / "bad.yaml"
|
||||
p.write_text("- 1\n- 2\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="yaml root must be a mapping"):
|
||||
load_yaml(str(p))
|
||||
|
||||
|
||||
def test_dump_yaml_roundtrip_smoke(tmp_path: Path):
|
||||
from argus.ray.yaml_io import dump_yaml, load_yaml
|
||||
|
||||
text = dump_yaml({"a": 1, "b": {"c": "d"}})
|
||||
assert "a: 1" in text
|
||||
|
||||
p = tmp_path / "x.yaml"
|
||||
p.write_text(text, encoding="utf-8")
|
||||
assert load_yaml(str(p))["b"]["c"] == "d"
|
||||
|
||||
Reference in New Issue
Block a user