NASP/argus-cluster
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v3.8 改成每个模型独立app，测试通过容器down， ray serve自动迁移模型

Browse Source
This commit is contained in:
yuyr
2026-01-08 17:33:16 +08:00
parent 686739fea2
commit 938f84f1ba
12 changed files with 743 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/mvp/py/argus/service/app.py
View File

@@ -91,19 +91,17 @@ def create_app(config_path: str) -> FastAPI:
def _serving_enabled() -> bool:
return bool(v2_cfg.serving.enabled)
def _openai_base_url(req: Request) -> str:
def _openai_base_url_for_model(req: Request, *, model_key: str) -> str:
# Prefer forwarded headers if present; otherwise fall back to Host.
host = req.headers.get("x-forwarded-host") or req.headers.get("host") or req.url.hostname or "127.0.0.1"
# Strip port if present (common for Host header).
hostname = host
if hostname.startswith("[") and "]" in hostname:
# IPv6 like: [::1]:8080
hostname = hostname.split("]")[0] + "]"
else:
hostname = hostname.split(":")[0]
scheme = req.headers.get("x-forwarded-proto") or req.url.scheme or "http"
port = int(v2_cfg.serving.serve.http_port)
return f"{scheme}://{hostname}:{port}/v1"
return f"{scheme}://{hostname}:{port}/serve/{model_key}/v1"
def _dump_yaml(obj: Any) -> str:
return yaml.safe_dump(obj, sort_keys=False)
@@ -122,8 +120,9 @@ def create_app(config_path: str) -> FastAPI:
gpus_per_replica = int(row.get("gpus_per_replica") or 0)
total_gpus = num_replicas * gpus_per_replica
model_id = str(row.get("model_id") or "")
model_key = str(row.get("model_key") or "")
return {
"model_key": str(row.get("model_key") or ""),
"model_key": model_key,
"user_id": str(row.get("user_id") or ""),
"model_id": model_id,
"model_id_suffix": str(row.get("model_id_suffix") or ""),
@@ -138,7 +137,7 @@ def create_app(config_path: str) -> FastAPI:
"updated_at": str(row.get("updated_at") or ""),
"deleted_at": row.get("deleted_at"),
"endpoint": {
"openai_base_url": _openai_base_url(req),
"openai_base_url": _openai_base_url_for_model(req, model_key=model_key),
"model": model_id,
},
}
@@ -688,7 +687,8 @@ def create_app(config_path: str) -> FastAPI:
out = [_serve_model_public(i, req=req) for i in items]
return {
"items": out,
"openai_base_url": _openai_base_url(req),
# Per-model route_prefix; see each item.endpoint.openai_base_url.
"openai_base_url": None,
"limit": lim,
"offset": off,
"has_more": bool(len(items) == lim),
@@ -773,7 +773,8 @@ def create_app(config_path: str) -> FastAPI:
raise HTTPException(status_code=400, detail="serving is not enabled")
return {
"enabled": True,
"openai_base_url": _openai_base_url(req),
# Per-model route_prefix; see /api/v2/serve/models items[].endpoint.openai_base_url.
"openai_base_url": None,
"http_port": int(v2_cfg.serving.serve.http_port),
"proxy_location": str(v2_cfg.serving.serve.proxy_location),
"accelerator_type": str(v2_cfg.serving.llm.accelerator_type),