151 lines
6.0 KiB
Python
151 lines
6.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import traceback
|
|
from dataclasses import dataclass
|
|
from typing import Any, Protocol
|
|
|
|
from argus.service.ray_resources import ClusterAvailable, get_cluster_available
|
|
|
|
from .config import V2Config
|
|
from .db import Db
|
|
from .serve_llm_config import build_llm_config_dict
|
|
from .serving_spec import ResolvedServingSpec
|
|
|
|
|
|
class ServeClient(Protocol):
|
|
def ensure_started(self) -> None: ...
|
|
|
|
def apply_app(self, *, app: Any, app_name: str, route_prefix: str = "/") -> Any: ...
|
|
|
|
def delete_app(self, *, app_name: str) -> Any: ...
|
|
|
|
def get_status(self) -> Any: ...
|
|
|
|
|
|
def _parse_engine_kwargs(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
raw = row.get("engine_kwargs_json")
|
|
if raw in (None, ""):
|
|
return None
|
|
try:
|
|
obj = json.loads(str(raw))
|
|
return obj if isinstance(obj, dict) else None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _row_to_resolved_spec(row: dict[str, Any]) -> ResolvedServingSpec:
|
|
return ResolvedServingSpec(
|
|
user_id=str(row["user_id"]),
|
|
model_id_suffix=str(row["model_id_suffix"]),
|
|
model_id_prefix=str(row["model_id_prefix"]),
|
|
model_id=str(row["model_id"]),
|
|
model_source=str(row["model_source"]),
|
|
num_replicas=int(row["num_replicas"]),
|
|
gpus_per_replica=int(row["gpus_per_replica"]),
|
|
engine_kwargs=_parse_engine_kwargs(row),
|
|
)
|
|
|
|
|
|
def _needed_model_gpus(row: dict[str, Any]) -> int:
|
|
return int(row.get("num_replicas") or 0) * int(row.get("gpus_per_replica") or 0)
|
|
|
|
|
|
@dataclass
|
|
class ServingReconciler:
|
|
"""
|
|
v3.8 (per-model apps): reconcile serve_models (SQLite) into per-model Ray Serve apps.
|
|
|
|
This reconciler is intentionally conservative:
|
|
- Only acts on models in states QUEUED/DELETING.
|
|
- Performs a minimal GPU precheck using ray available GPU totals (per-model).
|
|
- Writes events and state transitions for explainability.
|
|
"""
|
|
|
|
db: Db
|
|
v2_cfg: V2Config
|
|
ray_runtime_env_env_vars: dict[str, str]
|
|
serve_client: ServeClient
|
|
cpu_per_gpu: float = 1.0
|
|
get_available_fn: Any = get_cluster_available
|
|
|
|
def tick(self) -> None:
|
|
# Pick the next desired change.
|
|
change = self.db.pick_next_runnable_serve_change()
|
|
if not change:
|
|
return
|
|
|
|
model_key = str(change["model_key"])
|
|
state = str(change.get("state") or "")
|
|
|
|
# Ensure Ray (and Serve) can be started before doing anything else.
|
|
try:
|
|
self.serve_client.ensure_started()
|
|
except Exception as e:
|
|
self.db.append_serve_event(model_key=model_key, event_type="SERVE_START_ERROR", payload_json=repr(e))
|
|
return
|
|
|
|
# Deleting is per-model: remove the model's app by name. This avoids touching other models.
|
|
if state == "DELETING":
|
|
try:
|
|
self.db.append_serve_event(model_key=model_key, event_type="SERVE_DELETE_REQUESTED", payload_json=None)
|
|
self.serve_client.delete_app(app_name=model_key)
|
|
except Exception as e:
|
|
err = f"{type(e).__name__}: {e}"
|
|
tb = traceback.format_exc(limit=10)
|
|
self.db.set_serve_model_state(
|
|
model_key=model_key, state="FAILED", error_summary=err, event_type="SERVE_DELETE_FAILED", payload_json=tb
|
|
)
|
|
return
|
|
self.db.set_serve_model_state(model_key=model_key, state="DELETED", event_type="SERVE_DELETE_APPLIED")
|
|
return
|
|
|
|
# Precheck resources: per-model apply needs enough GPUs for this model only.
|
|
needed = _needed_model_gpus(change)
|
|
avail: ClusterAvailable = self.get_available_fn()
|
|
if float(avail.total_available_gpus) < float(needed):
|
|
msg = f"Insufficient GPUs: need {needed}, available {avail.total_available_gpus}"
|
|
self.db.append_serve_event(model_key=model_key, event_type="SERVE_PENDING_RESOURCES", payload_json=msg)
|
|
return
|
|
|
|
# Build the single-model LLM config (dict form for unit tests).
|
|
accelerator_type = str(self.v2_cfg.serving.llm.accelerator_type or "")
|
|
resolved = _row_to_resolved_spec(change)
|
|
llm_cfg_dicts = [
|
|
build_llm_config_dict(
|
|
resolved,
|
|
accelerator_type=accelerator_type,
|
|
runtime_env_env_vars=self.ray_runtime_env_env_vars,
|
|
cpu_per_gpu=self.cpu_per_gpu,
|
|
)
|
|
]
|
|
|
|
# Build a Ray Serve OpenAI-compatible app if Ray Serve LLM is available.
|
|
# Fall back to a plain dict so unit tests can run without real Ray Serve.
|
|
app_obj: Any
|
|
try:
|
|
from ray.serve.llm import LLMConfig, build_openai_app # type: ignore
|
|
|
|
llm_cfgs = [LLMConfig(**d) for d in llm_cfg_dicts]
|
|
app_obj = build_openai_app({"llm_configs": llm_cfgs})
|
|
except Exception as e:
|
|
self.db.append_serve_event(model_key=model_key, event_type="SERVE_LLM_IMPORT_ERROR", payload_json=repr(e))
|
|
app_obj = {"llm_configs": llm_cfg_dicts}
|
|
|
|
try:
|
|
self.db.append_serve_event(model_key=model_key, event_type="SERVE_APPLY_REQUESTED", payload_json="1")
|
|
self.serve_client.apply_app(app=app_obj, app_name=model_key, route_prefix=f"/serve/{model_key}")
|
|
except Exception as e:
|
|
err = f"{type(e).__name__}: {e}"
|
|
tb = traceback.format_exc(limit=10)
|
|
self.db.set_serve_model_state(model_key=model_key, state="FAILED", error_summary=err, event_type="SERVE_APPLY_FAILED", payload_json=tb)
|
|
return
|
|
|
|
# Mark as deploying; best-effort status probe can promote to RUNNING.
|
|
self.db.set_serve_model_state(model_key=model_key, state="DEPLOYING", event_type="SERVE_DEPLOYING")
|
|
try:
|
|
_ = self.serve_client.get_status()
|
|
self.db.set_serve_model_state(model_key=model_key, state="RUNNING", event_type="SERVE_RUNNING")
|
|
except Exception as e:
|
|
self.db.append_serve_event(model_key=model_key, event_type="SERVE_STATUS_ERROR", payload_json=repr(e))
|