From c7f16bb906887b0ae13770a5e26f5991c40cbfad Mon Sep 17 00:00:00 2001 From: yuyr Date: Wed, 24 Sep 2025 09:25:06 +0000 Subject: [PATCH] =?UTF-8?q?[#2]=20master=E6=A8=A1=E5=9D=97=E7=A6=BB?= =?UTF-8?q?=E7=BA=BF=E6=9E=84=E5=BB=BA=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/master/Dockerfile | 15 ++- src/master/README.md | 24 ++++- src/master/offline_wheels/.gitkeep | 0 src/master/scripts/build_images.sh | 26 ++++- src/master/scripts/prepare_offline_wheels.sh | 97 +++++++++++++++++++ src/master/tests/docker-compose.yml | 2 +- src/master/tests/scripts/00_e2e_test.sh | 2 +- .../tests/scripts/00_e2e_test_offline.sh | 16 +++ src/master/tests/scripts/01_up_master.sh | 6 +- 9 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 src/master/offline_wheels/.gitkeep create mode 100755 src/master/scripts/prepare_offline_wheels.sh create mode 100755 src/master/tests/scripts/00_e2e_test_offline.sh diff --git a/src/master/Dockerfile b/src/master/Dockerfile index 9ab73de..32c2e3d 100644 --- a/src/master/Dockerfile +++ b/src/master/Dockerfile @@ -3,6 +3,7 @@ FROM python:3.11-slim SHELL ["/bin/bash", "-c"] ARG PIP_INDEX_URL= +ARG USE_OFFLINE=0 ENV PIP_NO_CACHE_DIR=1 \ PYTHONUNBUFFERED=1 \ PYTHONPATH=/app @@ -10,12 +11,18 @@ ENV PIP_NO_CACHE_DIR=1 \ WORKDIR /app COPY requirements.txt ./ +COPY offline_wheels/ /opt/offline_wheels/ + RUN set -euxo pipefail \ - && python -m pip install --upgrade pip \ - && if [[ -n "$PIP_INDEX_URL" ]]; then \ - PIP_INDEX_URL="$PIP_INDEX_URL" python -m pip install -r requirements.txt; \ + && if [[ "$USE_OFFLINE" == "1" ]]; then \ + python -m pip install --no-index --find-links /opt/offline_wheels -r requirements.txt; \ else \ - python -m pip install -r requirements.txt; \ + python -m pip install --upgrade pip \ + && if [[ -n "$PIP_INDEX_URL" ]]; then \ + PIP_INDEX_URL="$PIP_INDEX_URL" python -m pip install -r requirements.txt; \ + else \ + python -m pip install -r requirements.txt; \ + fi; \ fi COPY app ./app diff --git a/src/master/README.md b/src/master/README.md index 2779e19..3e7cd8a 100644 --- a/src/master/README.md +++ b/src/master/README.md @@ -14,6 +14,22 @@ cd src/master ./scripts/build_images.sh # 生成 argus-master:dev 镜像 ``` +如需离线构建，先在有网环境运行准备脚本： + +```bash +cd src/master +./scripts/prepare_offline_wheels.sh --pip-version 25.2 # 可选 --clean +``` + +脚本会把 `requirements.txt` 及 pip 指定版本全部下载到 `offline_wheels/`。随后将源码目录（含该子目录）与基础镜像一并拷贝到内网，执行： + +```bash +cd src/master +./scripts/build_images.sh --offline --tag argus-master:dev +``` + +若内网缺少 `python:3.11-slim`，请提前在外网 `docker save` 后通过离线介质 `docker load`。 + 本仓库提供的端到端测试会使用 `src/master/tests/docker-compose.yml` 启动示例环境： ```bash @@ -124,6 +140,13 @@ cd src/master/tests - `GET /healthz`：进程存活检查。 - `GET /readyz`：数据库可用性检查（会尝试访问 `DB_PATH`）。 + +如需验证离线镜像，可使用自动化脚本： +```bash +cd src/master/tests +./scripts/00_e2e_test_offline.sh # 构建离线镜像并执行完整 E2E +``` + ## 端到端测试场景 执行 `src/master/tests/scripts/00_e2e_test.sh` 会串联以下用例（脚本 01–10）： @@ -146,4 +169,3 @@ cd src/master/tests - 测试用例中的 `tests/private/`、`tests/tmp/` 会随脚本自动清理，避免污染后续运行。 如需在生产环境运行，可将镜像推送到私有仓库，或参考测试 Compose 配置自行部署；只需确保上述环境变量在容器内正确设置即可。 - diff --git a/src/master/offline_wheels/.gitkeep b/src/master/offline_wheels/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/master/scripts/build_images.sh b/src/master/scripts/build_images.sh index 501ac02..ae587d5 100755 --- a/src/master/scripts/build_images.sh +++ b/src/master/scripts/build_images.sh @@ -2,13 +2,21 @@ set -euo pipefail usage() { - echo "Usage: $0 [--intranet] [--tag ]" >&2 + cat >&2 <<'USAGE' +Usage: $0 [--intranet] [--offline] [--tag ] + +Options: + --intranet 使用指定的 PyPI 镜像源（默认清华镜像）。 + --offline 完全离线构建，依赖 offline_wheels/ 目录中的离线依赖包。 + --tag 自定义镜像标签，默认 argus-master:dev。 +USAGE } SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" IMAGE_TAG="${IMAGE_TAG:-argus-master:dev}" BUILD_ARGS=() +OFFLINE_MODE=0 while [[ "$#" -gt 0 ]]; do case "$1" in @@ -17,6 +25,11 @@ while [[ "$#" -gt 0 ]]; do BUILD_ARGS+=("--build-arg" "PIP_INDEX_URL=${INTRANET_INDEX}") shift ;; + --offline) + OFFLINE_MODE=1 + BUILD_ARGS+=("--build-arg" "USE_OFFLINE=1") + shift + ;; --tag) [[ $# -ge 2 ]] || { usage; exit 1; } IMAGE_TAG="$2" @@ -34,6 +47,17 @@ while [[ "$#" -gt 0 ]]; do esac done +if [[ "$OFFLINE_MODE" -eq 1 ]]; then + WHEELS_DIR="$PROJECT_ROOT/offline_wheels" + if [[ ! -d "$WHEELS_DIR" ]]; then + echo "[ERROR] offline_wheels 目录不存在: $WHEELS_DIR" >&2 + exit 1 + fi + if ! find "$WHEELS_DIR" -maxdepth 1 -type f -name '*.whl' -print -quit >/dev/null; then + echo "[WARN] offline_wheels 目录为空，请确保已提前下载所需的 wheel 包" >&2 + fi +fi + echo "[INFO] Building image $IMAGE_TAG" docker build "${BUILD_ARGS[@]}" -t "$IMAGE_TAG" "$PROJECT_ROOT" echo "[OK] Image $IMAGE_TAG built" diff --git a/src/master/scripts/prepare_offline_wheels.sh b/src/master/scripts/prepare_offline_wheels.sh new file mode 100755 index 0000000..08037ed --- /dev/null +++ b/src/master/scripts/prepare_offline_wheels.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: $0 [--pip-version ] [--clean] [--local] + +Options: + --pip-version 额外下载指定版本的 pip wheel（例如 25.2）。 + --clean 清理 offline_wheels/*.whl 后重新下载。 + --local 使用本地 python 执行下载（默认通过 docker python:3.11-slim）。 +USAGE +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +REQUIREMENTS_FILE="$PROJECT_ROOT/requirements.txt" +WHEEL_DIR="$PROJECT_ROOT/offline_wheels" +PIP_VERSION="" +CLEAN=0 +USE_LOCAL=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --pip-version) + [[ $# -ge 2 ]] || { usage; exit 1; } + PIP_VERSION="$2" + shift 2 + ;; + --clean) + CLEAN=1 + shift + ;; + --local) + USE_LOCAL=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage + exit 1 + ;; + esac + done + +if [[ ! -f "$REQUIREMENTS_FILE" ]]; then + echo "[ERROR] requirements.txt not found at $REQUIREMENTS_FILE" >&2 + exit 1 +fi + +mkdir -p "$WHEEL_DIR" + +if [[ "$CLEAN" -eq 1 ]]; then + echo "[INFO] Cleaning existing wheels in $WHEEL_DIR" + find "$WHEEL_DIR" -maxdepth 1 -type f -name '*.whl' -delete +fi + +run_with_python() { + local cmd=("python" "-m" "pip" "$@") + eval "${cmd[@]}" +} + +if [[ "$USE_LOCAL" -eq 1 ]]; then + PYTHON_BIN=${PYTHON_BIN:-python3} + if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "[ERROR] $PYTHON_BIN not found" >&2 + exit 1 + fi + echo "[INFO] Using local python ($PYTHON_BIN) to download wheels" + "$PYTHON_BIN" -m pip download -r "$REQUIREMENTS_FILE" -d "$WHEEL_DIR" + if [[ -n "$PIP_VERSION" ]]; then + "$PYTHON_BIN" -m pip download "pip==${PIP_VERSION}" -d "$WHEEL_DIR" + fi +else + if ! command -v docker >/dev/null 2>&1; then + echo "[ERROR] docker not found; rerun with --local or安装 docker" >&2 + exit 1 + fi + echo "[INFO] Using docker image python:3.11-slim 下载 wheel" + docker run --rm \ + -v "$WHEEL_DIR":/wheels \ + -v "$REQUIREMENTS_FILE":/tmp/requirements.txt \ + python:3.11-slim \ + bash -c "set -euo pipefail && python -m pip install --upgrade pip && python -m pip download -r /tmp/requirements.txt -d /wheels" + if [[ -n "$PIP_VERSION" ]]; then + docker run --rm \ + -v "$WHEEL_DIR":/wheels \ + python:3.11-slim \ + bash -c "set -euo pipefail && python -m pip download pip==${PIP_VERSION} -d /wheels" + fi +fi + +echo "[INFO] Offline wheels prepared at $WHEEL_DIR" diff --git a/src/master/tests/docker-compose.yml b/src/master/tests/docker-compose.yml index ffb9b4b..8c24661 100644 --- a/src/master/tests/docker-compose.yml +++ b/src/master/tests/docker-compose.yml @@ -1,6 +1,6 @@ services: master: - image: argus-master:dev + image: ${MASTER_IMAGE_TAG:-argus-master:dev} container_name: argus-master-e2e environment: - OFFLINE_THRESHOLD_SECONDS=6 diff --git a/src/master/tests/scripts/00_e2e_test.sh b/src/master/tests/scripts/00_e2e_test.sh index 9d142db..520d3a9 100755 --- a/src/master/tests/scripts/00_e2e_test.sh +++ b/src/master/tests/scripts/00_e2e_test.sh @@ -17,7 +17,7 @@ SCRIPTS=( for script in "${SCRIPTS[@]}"; do echo "[TEST] Running $script" - "$SCRIPT_DIR/$script" + MASTER_IMAGE_TAG="${MASTER_IMAGE_TAG:-argus-master:dev}" "$SCRIPT_DIR/$script" echo "[TEST] $script completed" echo done diff --git a/src/master/tests/scripts/00_e2e_test_offline.sh b/src/master/tests/scripts/00_e2e_test_offline.sh new file mode 100755 index 0000000..1c3fc0d --- /dev/null +++ b/src/master/tests/scripts/00_e2e_test_offline.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODULE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MASTER_ROOT="$(cd "$MODULE_ROOT/.." && pwd)" + +# 准备离线依赖并构建镜像 +pushd "$MASTER_ROOT" >/dev/null +./scripts/prepare_offline_wheels.sh --clean --pip-version 25.2 +./scripts/build_images.sh --offline --tag argus-master:offline +popd >/dev/null + +# 使用离线镜像执行既有端到端用例 +MASTER_IMAGE_TAG="argus-master:offline" ./scripts/00_e2e_test.sh + diff --git a/src/master/tests/scripts/01_up_master.sh b/src/master/tests/scripts/01_up_master.sh index 8b2d385..10f6dc2 100755 --- a/src/master/tests/scripts/01_up_master.sh +++ b/src/master/tests/scripts/01_up_master.sh @@ -29,13 +29,9 @@ mkdir -p "$PRIVATE_ROOT/argus/master" mkdir -p "$PRIVATE_ROOT/argus/metric/prometheus" mkdir -p "$TMP_ROOT" -pushd "$MODULE_ROOT" >/dev/null -./scripts/build_images.sh --tag argus-master:dev -popd >/dev/null - pushd "$TEST_ROOT" >/dev/null compose down --remove-orphans || true -compose up -d +MASTER_IMAGE_TAG="${MASTER_IMAGE_TAG:-argus-master:dev}" compose up -d popd >/dev/null echo "[INFO] Master container is up on http://localhost:31300"