v3.8 改成每个模型独立app，测试通过容器down， ray serve自动迁移模型

src/mvp/py/argus/service/serving_reconciler.py

@@ -18,6 +18,8 @@ class ServeClient(Protocol):
 
     def apply_app(self, *, app: Any, app_name: str, route_prefix: str = "/") -> Any: ...
 
+    def delete_app(self, *, app_name: str) -> Any: ...
+
     def get_status(self) -> Any: ...
 
 
@@ -45,21 +47,18 @@ def _row_to_resolved_spec(row: dict[str, Any]) -> ResolvedServingSpec:
     )
 
 
-def _needed_total_gpus(rows: list[dict[str, Any]]) -> int:
-    total = 0
-    for r in rows:
-        total += int(r.get("num_replicas") or 0) * int(r.get("gpus_per_replica") or 0)
-    return total
+def _needed_model_gpus(row: dict[str, Any]) -> int:
+    return int(row.get("num_replicas") or 0) * int(row.get("gpus_per_replica") or 0)
 
 
 @dataclass
 class ServingReconciler:
     """
-    v3.8: reconcile declared serve_models (SQLite) into a multi-model Ray Serve app.
+    v3.8 (per-model apps): reconcile serve_models (SQLite) into per-model Ray Serve apps.
 
     This reconciler is intentionally conservative:
     - Only acts on models in states QUEUED/DELETING.
-    - Performs a minimal GPU precheck using ray available GPU totals.
+    - Performs a minimal GPU precheck using ray available GPU totals (per-model).
     - Writes events and state transitions for explainability.
     """
 
@@ -67,8 +66,6 @@ class ServingReconciler:
     v2_cfg: V2Config
     ray_runtime_env_env_vars: dict[str, str]
     serve_client: ServeClient
-    app_name: str = "argus_llm_app"
-    route_prefix: str = "/"
     cpu_per_gpu: float = 1.0
     get_available_fn: Any = get_cluster_available
 
@@ -88,33 +85,40 @@ class ServingReconciler:
             self.db.append_serve_event(model_key=model_key, event_type="SERVE_START_ERROR", payload_json=repr(e))
             return
 
-        # Desired set: all non-deleted models except those marked DELETING.
-        all_rows = self.db.list_all_serve_models(include_deleted=False, limit=5000, offset=0)
-        # FAILED models are not part of the desired running set. A user can PATCH to
-        # re-queue a failed model (e.g., after fixing env/deps) which will move it back to QUEUED.
-        desired_rows = [r for r in all_rows if str(r.get("state") or "") not in ("DELETING", "DELETED", "FAILED")]
+        # Deleting is per-model: remove the model's app by name. This avoids touching other models.
+        if state == "DELETING":
+            try:
+                self.db.append_serve_event(model_key=model_key, event_type="SERVE_DELETE_REQUESTED", payload_json=None)
+                self.serve_client.delete_app(app_name=model_key)
+            except Exception as e:
+                err = f"{type(e).__name__}: {e}"
+                tb = traceback.format_exc(limit=10)
+                self.db.set_serve_model_state(
+                    model_key=model_key, state="FAILED", error_summary=err, event_type="SERVE_DELETE_FAILED", payload_json=tb
+                )
+                return
+            self.db.set_serve_model_state(model_key=model_key, state="DELETED", event_type="SERVE_DELETE_APPLIED")
+            return
 
-        # Precheck resources: multi-model app apply needs enough GPUs for the whole desired set.
-        needed = _needed_total_gpus(desired_rows)
+        # Precheck resources: per-model apply needs enough GPUs for this model only.
+        needed = _needed_model_gpus(change)
         avail: ClusterAvailable = self.get_available_fn()
         if float(avail.total_available_gpus) < float(needed):
             msg = f"Insufficient GPUs: need {needed}, available {avail.total_available_gpus}"
             self.db.append_serve_event(model_key=model_key, event_type="SERVE_PENDING_RESOURCES", payload_json=msg)
             return
 
-        # Build per-model LLM configs (dict form in M4).
-        llm_cfg_dicts: list[dict[str, Any]] = []
+        # Build the single-model LLM config (dict form for unit tests).
         accelerator_type = str(self.v2_cfg.serving.llm.accelerator_type or "")
-        for r in desired_rows:
-            resolved = _row_to_resolved_spec(r)
-            llm_cfg_dicts.append(
-                build_llm_config_dict(
-                    resolved,
-                    accelerator_type=accelerator_type,
-                    runtime_env_env_vars=self.ray_runtime_env_env_vars,
-                    cpu_per_gpu=self.cpu_per_gpu,
-                )
+        resolved = _row_to_resolved_spec(change)
+        llm_cfg_dicts = [
+            build_llm_config_dict(
+                resolved,
+                accelerator_type=accelerator_type,
+                runtime_env_env_vars=self.ray_runtime_env_env_vars,
+                cpu_per_gpu=self.cpu_per_gpu,
             )
+        ]
 
         # Build a Ray Serve OpenAI-compatible app if Ray Serve LLM is available.
         # Fall back to a plain dict so unit tests can run without real Ray Serve.
@@ -129,19 +133,14 @@ class ServingReconciler:
             app_obj = {"llm_configs": llm_cfg_dicts}
 
         try:
-            self.db.append_serve_event(model_key=model_key, event_type="SERVE_APPLY_REQUESTED", payload_json=str(len(llm_cfg_dicts)))
-            self.serve_client.apply_app(app=app_obj, app_name=self.app_name, route_prefix=self.route_prefix)
+            self.db.append_serve_event(model_key=model_key, event_type="SERVE_APPLY_REQUESTED", payload_json="1")
+            self.serve_client.apply_app(app=app_obj, app_name=model_key, route_prefix=f"/serve/{model_key}")
         except Exception as e:
             err = f"{type(e).__name__}: {e}"
             tb = traceback.format_exc(limit=10)
             self.db.set_serve_model_state(model_key=model_key, state="FAILED", error_summary=err, event_type="SERVE_APPLY_FAILED", payload_json=tb)
             return
 
-        # Apply succeeded. Update the changing model's state.
-        if state == "DELETING":
-            self.db.set_serve_model_state(model_key=model_key, state="DELETED", event_type="SERVE_DELETE_APPLIED")
-            return
-
         # Mark as deploying; best-effort status probe can promote to RUNNING.
         self.db.set_serve_model_state(model_key=model_key, state="DEPLOYING", event_type="SERVE_DEPLOYING")
         try: