From 3709acd92a0a4ca64e56a9f4728b0d6423c963f6 Mon Sep 17 00:00:00 2001 From: Aron Xu Date: Tue, 11 Aug 2026 17:37:33 +0800 Subject: [PATCH 01/22] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=88=86=E5=B1=82?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E6=8C=87=E6=A0=87=E9=87=87=E9=9B=86=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 只依赖已有数据源,不引入新组件: - 各 Squid 的 access.log(命中/字节/客户端/bump-splice) - HAProxy stats CSV :8404(后端分发与健康) - cgroup cpu.stat(CPU 秒 → 每 GB CPU 成本) 按 begin/end 框定区间输出四层指标: Squid 缓存效果 / SSL Bump CPU 成本 / HAProxy 分发健康 / 客户端归因。 采集上有三个坑,都在脚本里处理了: 1. 必须框定区间。累计值把预热、健康检查、历史负载混在一起, 算出来的命中率和 CPU/GB 没有意义。 2. 健康检查要剔除。每 3 秒 × 3 后端 × 2 节点,日志里是 NONE_NONE/400、 方法为 -,不滤掉会把请求数和命中率彻底冲淡。 3. CONNECT 隧道不是对象请求。实测 bump 的 CONNECT 记录是 NONE_NONE/200 bytes=0,只是建隧道,真正负载会作为解密后的 GET https://... 再记一行 —— 混进去会凭空翻倍请求数并稀释命中率。 splice 则相反,是 TCP_TUNNEL 带真实字节且不可能命中。 最初的实现把这两者搞混了,4 次下载被记成 8 个请求。 实测(4 个从未取过的包,分别跑纯 MISS 与纯 HIT 两个窗口): 纯 MISS: 命中率 0% , 44.42 CPU秒/GB 纯 HIT : 命中率 76.4%, 37.13 CPU秒/GB 注意这个小样本(4 请求、低并发)不足以复现 k8s 压测里 「纯 HIT 比 MISS 更吃 CPU」的结论 —— 那是 N=120 高并发下 TLS 批量加密 打满 CPU 的效应,低并发时开销主要在握手和证书生成。 脚本的作用是提供采集手段,系数仍需在目标并发下实测。 归因层依赖 PROXY protocol(见 PR #9),未生效时脚本会给出提示。 Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 1 + README.md | 35 ++++++++++ scripts/metrics.sh | 162 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100755 scripts/metrics.sh diff --git a/.gitignore b/.gitignore index bb5a2bd..84c4a61 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ test-report.txt result.json k8s/*.csv buildkit/src/ +.metrics-state diff --git a/README.md b/README.md index 41e7827..1b99604 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,41 @@ cd buildkit && ./run-tests.sh # 运行 BuildKit 扩展测试(自动起停 buil 详见 `buildkit/README.md` 与 `buildkit/DESIGN.md`。依赖 BuildKit [PR #1](https://github.com/TommyLike/buildkit/pull/1)(`feature/upstream-proxy-config` 分支)。 +## 分层性能指标 + +`scripts/metrics.sh` 按区间采集分层指标,只依赖已有数据源(各 Squid 的 `access.log`、 +HAProxy stats CSV、cgroup `cpu.stat`),不引入任何新组件: + +```bash +./scripts/metrics.sh begin # 打基线 +<跑你的负载> +./scripts/metrics.sh end # 输出这段区间的分层指标 +``` + +输出四层:**Squid**(请求/字节命中率、省下的回源字节)、**SSL Bump**(bump/splice 隧道数、 +每 GB CPU 秒)、**HAProxy**(后端分发、健康、检查失败)、**归因**(按客户端的请求数与字节数)。 + +三个采集上的坑,脚本里已经处理: + +- **必须框定区间**。累计值会把预热、健康检查、历史负载混在一起,命中率和 CPU/GB 都没有意义。 +- **健康检查要剔除**。每 3 秒 × 3 后端 × 2 节点,不滤掉会把请求数和命中率彻底冲淡 + (日志里是 `NONE_NONE/400`、方法为 `-`)。 +- **CONNECT 隧道不是对象请求**。`bump` 的 CONNECT 记录是 `NONE_NONE/200 bytes=0`, + 只是建隧道,真正的负载会作为解密后的 `GET https://...` 再记一行;混进去会凭空翻倍 + 请求数并稀释命中率。`splice` 则相反,是 `TCP_TUNNEL` 带真实字节且不可能命中。 + +> **归因层需要 PROXY protocol**。HAProxy 是 TCP 模式转发,默认情况下 Squid 看到的客户端 +> 永远是 HAProxy 节点,按 worker/job 的归因做不出来。脚本会在检测到这种情况时给出提示。 + +### 关于「命中率」不是唯一指标 + +合并进来的 K8s 压测数据(`reports/stress-benchmark-20260811.md`)显示:N=120 时 +**纯 HIT 比纯 MISS 更吃 CPU**(199m vs 88m),HIT 反而更慢(104s vs 67s)—— +因为命中要对内容重新做 TLS 加密,而回源只是在等被限速的源站。 + +所以容量推导的关键系数是**每 GB 服务消耗的 CPU 秒数**,并且要按 HIT/MISS、bump/splice 分开测: +分别跑一个「纯 HIT」窗口和一个「纯 MISS」窗口再比较。单看命中率会得出错误的扩容结论。 + ## 其他部分 | 部分 | 位置 | 说明 | diff --git a/scripts/metrics.sh b/scripts/metrics.sh new file mode 100755 index 0000000..1c18e1f --- /dev/null +++ b/scripts/metrics.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# 分层性能指标采集 —— 只依赖两个已有数据源,不引入任何新组件: +# 1) 各 Squid 的 access.log (命中/字节/客户端/bump-splice) +# 2) HAProxy stats CSV (:8404) (后端分发/健康状态) +# 3) cgroup cpu.stat (CPU 秒,用于算每 GB 的 CPU 成本) +# +# 用法: +# ./scripts/metrics.sh begin # 打基线 +# <跑你的负载> +# ./scripts/metrics.sh end # 输出这段区间的分层指标 +# ./scripts/metrics.sh now # 不打基线,直接看累计值 +# +# 为什么要按区间而不是看累计: 累计值会把预热、健康检查、历史负载混在一起, +# 得出的命中率和 CPU/GB 都没有意义。必须框定一个已知负载的窗口。 +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" +cd "$ROOT" +STATE="$ROOT/.metrics-state" +SQUIDS="squid1 squid2 squid3" + +# access.log 字段(squid 格式): +# $1 时间戳 $2 耗时ms $3 客户端 $4 结果码/状态 $5 字节 +# $6 方法 $7 URL $8 ident $9 层级/对端 $10 类型 +# +# $6 == "-" 的是 HAProxy 健康检查(NONE_NONE/400),必须剔除: +# 每 3 秒 × 3 后端 × 2 节点,不滤掉会把请求数和命中率彻底冲淡。 +AWK_AGG=' +$6 == "-" { next } # HAProxy 健康检查 +{ cli[$3] += $5; clir[$3]++ } + +# CONNECT 行是隧道记录,不是可缓存对象,必须单独统计: +# bump : NONE_NONE/200 bytes=0 —— 只是建隧道,真正的负载会作为解密后的 +# GET https://... 再记一行。混进 req 会凭空翻倍请求数并稀释命中率。 +# splice: TCP_TUNNEL/... 带真实字节 —— 未解密直通,Squid 看不到内容,不可能命中。 +$6 == "CONNECT" { + tunnel++ + if ($4 ~ /^TCP_TUNNEL/) { splice++; sbytes += $5 } + else { bumped++ } + next +} + +# 以下只统计可缓存的对象请求 +{ + req++; bytes += $5 + if ($4 ~ /TCP_(MEM_)?HIT/) { hit++; hitb += $5 } + else if ($4 ~ /TCP_(REFRESH|IMS)_HIT/) { rhit++; hitb += $5 } + else { miss++; missb += $5 } +} +END { + printf "req=%d hit=%d rhit=%d miss=%d bytes=%d hitb=%d missb=%d bumped=%d splice=%d sbytes=%d tunnel=%d\n", + req, hit, rhit, miss, bytes, hitb, missb, bumped, splice, sbytes, tunnel + for (c in cli) printf "CLIENT %s %d %d\n", c, clir[c], cli[c] +}' + +cpu_usec() { docker exec "$1" sh -c 'awk "/^usage_usec/{print \$2}" /sys/fs/cgroup/cpu.stat' 2>/dev/null || echo 0; } +log_lines() { docker exec "$1" sh -c 'wc -l < /var/log/squid/access.log' 2>/dev/null | tr -d ' ' || echo 0; } + +vip_holder() { + for n in haproxy-node1 haproxy-node2; do + docker exec "$n" ip -4 addr show eth0 2>/dev/null | grep -q 172.30.0.100 && { echo "$n"; return; } + done + echo haproxy-node1 +} + +do_begin() { + : > "$STATE" + for s in $SQUIDS; do echo "$s $(log_lines "$s") $(cpu_usec "$s")" >> "$STATE"; done + echo "TS $(date +%s)" >> "$STATE" + echo "基线已记录 ($STATE)。跑完负载后执行: ./scripts/metrics.sh end" +} + +# 汇总区间内的 access.log。$1=起始行(0 表示全部) +collect() { + local mode="$1" + for s in $SQUIDS; do + local from=0 + [ "$mode" = "delta" ] && from=$(awk -v s="$s" '$1==s{print $2}' "$STATE" 2>/dev/null) + from=${from:-0} + docker exec "$s" sh -c "tail -n +$((from + 1)) /var/log/squid/access.log" 2>/dev/null + done | awk "$AWK_AGG" +} + +report() { + local mode="$1" out + out=$(collect "$mode") + local agg; agg=$(echo "$out" | grep -v '^CLIENT ') + eval "$(echo "$agg" | tr ' ' '\n' | sed 's/^/M_/')" 2>/dev/null + + local cpu_s=0 elapsed=0 + for s in $SQUIDS; do + local now_us base_us=0 + now_us=$(cpu_usec "$s") + [ "$mode" = "delta" ] && base_us=$(awk -v s="$s" '$1==s{print $3}' "$STATE" 2>/dev/null) + cpu_s=$(awk -v a="$cpu_s" -v n="${now_us:-0}" -v b="${base_us:-0}" 'BEGIN{printf "%.3f", a+(n-b)/1000000}') + done + if [ "$mode" = "delta" ]; then + local t0; t0=$(awk '$1=="TS"{print $2}' "$STATE") + elapsed=$(( $(date +%s) - ${t0:-0} )) + fi + + local req=${M_req:-0} hit=${M_hit:-0} rhit=${M_rhit:-0} miss=${M_miss:-0} + local bytes=${M_bytes:-0} hitb=${M_hitb:-0} missb=${M_missb:-0} + local bumped=${M_bumped:-0} splice=${M_splice:-0} sbytes=${M_sbytes:-0} tunnel=${M_tunnel:-0} + + echo "============================================" + echo " 分层性能指标 ($([ "$mode" = delta ] && echo "区间 ${elapsed}s" || echo "累计"))" + echo "============================================" + + echo "" + echo "── Squid 层: 缓存效果 (只统计可缓存对象请求) ──" + if [ "$req" -eq 0 ]; then + echo " (区间内没有对象请求; 健康检查与 CONNECT 隧道已剔除)" + else + awk -v r="$req" -v h="$hit" -v rh="$rhit" -v m="$miss" -v b="$bytes" -v hb="$hitb" -v mb="$missb" 'BEGIN{ + printf " 请求数 %d (HIT %d / REFRESH_HIT %d / MISS %d)\n", r, h, rh, m + printf " 请求命中率 %.1f%%\n", (h+rh)*100/r + printf " 字节命中率 %.1f%% ← 省下的出向带宽看这个\n", (b>0? hb*100/b : 0) + printf " 服务字节 %.2f GB\n", b/1073741824 + printf " 回源字节 %.2f GB\n", mb/1073741824 + printf " 省下回源 %.2f GB\n", hb/1073741824 + }' + fi + + echo "" + echo "── SSL Bump 层: CPU 成本 ──" + awk -v b="$bumped" -v sp="$splice" -v sb="$sbytes" -v t="$tunnel" 'BEGIN{ + printf " 隧道 %d 条: 解密(bump) %d / 直通(splice) %d", t, b, sp + if (sp > 0) printf " 直通字节 %.2f GB(不可能命中)", sb/1073741824 + printf "\n" + }' + awk -v c="$cpu_s" -v b="$bytes" 'BEGIN{ + printf " CPU 合计 %.2f 秒 (三副本)\n", c + if (b > 0) printf " 每 GB CPU %.2f 秒/GB ← 容量推导的关键系数\n", c/(b/1073741824) + else printf " 每 GB CPU n/a (无流量)\n" + }' + echo " 注意: 纯 HIT 场景 CPU 高于纯 MISS(命中要重新做 TLS 加密,回源只是等 I/O)。" + echo " 要拿到可用的系数,请分别跑「纯 HIT」与「纯 MISS」两个窗口再比较。" + + echo "" + echo "── HAProxy 层: 分发与健康 ──" + local node; node=$(vip_holder) + echo " VIP 持有者: $node" + docker exec "$node" curl -s 'http://127.0.0.1:8404/stats;csv' 2>/dev/null \ + | awk -F, '$1=="squid_pool" && $2 ~ /^s[0-9]+$/ { + printf " %-4s 状态=%-5s 累计会话=%-8s 检查失败=%-4s 不可用秒=%s\n", $2, $18, $8, $22, $24 }' + + echo "" + echo "── 归因层: 客户端分布 (需要 PROXY protocol) ──" + echo "$out" | grep '^CLIENT ' | sort -k4 -rn | head -8 | awk '{ + printf " %-16s 请求 %-6s 字节 %.2f MB\n", $2, $3, $4/1048576 }' + echo "$out" | grep -q '^CLIENT 172\.30\.0\.2[12] ' && \ + echo " ⚠ 客户端仍显示为 HAProxy 节点 —— PROXY protocol 未生效,归因不可用" + echo "" +} + +case "${1:-now}" in + begin) do_begin ;; + end) [ -f "$STATE" ] || { echo "没有基线,先跑 ./scripts/metrics.sh begin"; exit 1; }; report delta ;; + now) report full ;; + *) echo "用法: $0 {begin|end|now}"; exit 1 ;; +esac From 72df79b47ac09026c3610a3497175f74188125fd Mon Sep 17 00:00:00 2001 From: Aron Xu Date: Tue, 11 Aug 2026 17:51:58 +0800 Subject: [PATCH 02/22] =?UTF-8?q?CPU=20=E6=88=90=E6=9C=AC=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E4=B8=A4=E9=A1=B9=E6=A8=A1=E5=9E=8B,=E5=B9=B6?= =?UTF-8?q?=E6=89=A3=E9=99=A4=E7=A9=BA=E9=97=B2=E5=9F=BA=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原先只报一个「每 GB CPU」,这个数不可信,有两个问题: 1. CPU 不与字节成正比。它由「每请求固定成本」(TLS 握手、证书生成、缓存查找) 和「每字节边际成本」(加密、I/O)两项构成。只报每 GB 会把固定成本摊进字节, 于是同一套环境下这个数随对象大小漂移 —— 实测 14MB 对象约 28 秒/GB, 58KB-6MB 混合约 37-44 秒/GB,相差 1.6 倍。当成常数用会算错容量。 2. 窗口里的空闲 CPU 被算进了请求成本。Squid 零流量时也在烧 CPU: 健康检查(每3秒×3后端×2节点)、日志写入、缓存索引维护。 实测空闲基线约 0.004 CPU秒/秒(三副本合计)。窗口越稀疏(比如里面有大量 docker run 启动等待时间),这部分占比越高,实测能占到测量值的三分之一。 改法: - 新增 metrics.sh baseline [秒]: 零负载时测一次空闲 CPU 速率,存 .metrics-baseline - end 时按窗口时长扣除空闲基线,输出「原始 − 基线 = 归因」 - 拆成两项输出: 每请求 CPU / 每 GB CPU, 并同时给出平均对象大小 - 明确标注每 GB CPU 不是常数, 并给出容量公式: 核数 ≈ 请求速率 × 每请求CPU + 吞吐(GB/s) × 每GB CPU 拟合这两项需要在不同对象大小/并发下各跑一个窗口 顺带纠正之前基于这个指标得出的结论: 早先用「纯 MISS 44.42 秒/GB vs 纯 HIT 37.13 秒/GB」比较两者 CPU 成本是无效的 —— 两个窗口对象集不同, 差值又落在空闲基线污染的范围内,不足以支撑任何 HIT/MISS 结论。 低并发本来也测不出 k8s 压测里 N=120 时「HIT 比 MISS 更吃 CPU」的效应, 脚本里已加注,避免再用小样本去推翻高并发结论。 Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 1 + README.md | 23 +++++++++++++--- scripts/metrics.sh | 65 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 84c4a61..b5c7f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ result.json k8s/*.csv buildkit/src/ .metrics-state +.metrics-baseline diff --git a/README.md b/README.md index 1b99604..44b2ecc 100644 --- a/README.md +++ b/README.md @@ -153,13 +153,14 @@ cd buildkit && ./run-tests.sh # 运行 BuildKit 扩展测试(自动起停 buil HAProxy stats CSV、cgroup `cpu.stat`),不引入任何新组件: ```bash +./scripts/metrics.sh baseline # 零负载时跑一次,测空闲 CPU 基线(可复用) ./scripts/metrics.sh begin # 打基线 <跑你的负载> ./scripts/metrics.sh end # 输出这段区间的分层指标 ``` 输出四层:**Squid**(请求/字节命中率、省下的回源字节)、**SSL Bump**(bump/splice 隧道数、 -每 GB CPU 秒)、**HAProxy**(后端分发、健康、检查失败)、**归因**(按客户端的请求数与字节数)。 +CPU 两项成本)、**HAProxy**(后端分发、健康、检查失败)、**归因**(按客户端的请求数与字节数)。 三个采集上的坑,脚本里已经处理: @@ -178,9 +179,25 @@ HAProxy stats CSV、cgroup `cpu.stat`),不引入任何新组件: 合并进来的 K8s 压测数据(`reports/stress-benchmark-20260811.md`)显示:N=120 时 **纯 HIT 比纯 MISS 更吃 CPU**(199m vs 88m),HIT 反而更慢(104s vs 67s)—— 因为命中要对内容重新做 TLS 加密,而回源只是在等被限速的源站。 +所以只看命中率会得出错误的扩容结论,还必须看 CPU 成本。 -所以容量推导的关键系数是**每 GB 服务消耗的 CPU 秒数**,并且要按 HIT/MISS、bump/splice 分开测: -分别跑一个「纯 HIT」窗口和一个「纯 MISS」窗口再比较。单看命中率会得出错误的扩容结论。 +### CPU 成本是两项,不是一个「每 GB」常数 + +CPU 消耗 = **每请求固定成本**(TLS 握手、证书生成、缓存查找)+ **每字节边际成本**(加密、I/O)。 +只报「每 GB CPU」会把固定成本摊进字节里,于是同一套环境下这个数会随对象大小漂移 —— +实测 14MB 对象约 28 秒/GB,而 58KB–6MB 混合约 37–44 秒/GB,相差 1.6 倍。 + +容量估算要用两项模型: + +``` +核数 ≈ 请求速率 × 每请求CPU + 吞吐(GB/s) × 每GB CPU +``` + +要拟合这两项,需要在**不同对象大小/并发**下各跑一个窗口再比较,单个窗口拟合不出来。 + +另外必须先跑 `metrics.sh baseline`:Squid 零流量时也在烧 CPU(健康检查每 3 秒 × 3 后端 × 2 节点、 +日志写入、缓存索引维护)。窗口越稀疏——比如里面有大量 `docker run` 启动等待——这部分占比越高, +实测能占到测量值的三分之一。不扣掉会显著高估 CPU 成本,也会让两个窗口之间的比较失去意义。 ## 其他部分 diff --git a/scripts/metrics.sh b/scripts/metrics.sh index 1c18e1f..7ff1b20 100755 --- a/scripts/metrics.sh +++ b/scripts/metrics.sh @@ -5,18 +5,20 @@ # 3) cgroup cpu.stat (CPU 秒,用于算每 GB 的 CPU 成本) # # 用法: -# ./scripts/metrics.sh begin # 打基线 +# ./scripts/metrics.sh baseline [秒] # 测空闲 CPU 基线(零负载时跑一次,可复用) +# ./scripts/metrics.sh begin # 打基线 # <跑你的负载> -# ./scripts/metrics.sh end # 输出这段区间的分层指标 -# ./scripts/metrics.sh now # 不打基线,直接看累计值 +# ./scripts/metrics.sh end # 输出这段区间的分层指标 +# ./scripts/metrics.sh now # 不打基线,直接看累计值 # # 为什么要按区间而不是看累计: 累计值会把预热、健康检查、历史负载混在一起, -# 得出的命中率和 CPU/GB 都没有意义。必须框定一个已知负载的窗口。 +# 得出的命中率和 CPU 成本都没有意义。必须框定一个已知负载的窗口。 set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" ROOT="$(dirname "$SCRIPT_DIR")" cd "$ROOT" STATE="$ROOT/.metrics-state" +BASELINE_FILE="$ROOT/.metrics-baseline" SQUIDS="squid1 squid2 squid3" # access.log 字段(squid 格式): @@ -68,6 +70,20 @@ do_begin() { for s in $SQUIDS; do echo "$s $(log_lines "$s") $(cpu_usec "$s")" >> "$STATE"; done echo "TS $(date +%s)" >> "$STATE" echo "基线已记录 ($STATE)。跑完负载后执行: ./scripts/metrics.sh end" + [ -f "$BASELINE_FILE" ] || echo "提示: 尚未测空闲基线,CPU 归因会偏高。先跑一次 $0 baseline" +} + +# 空闲 CPU 基线。Squid 即使零流量也在烧 CPU: 健康检查(每3秒×3后端×2节点)、 +# 日志写入、缓存索引维护。窗口越稀疏(比如里面有大量 docker run 启动等待时间), +# 这部分占比越高 —— 实测能占到测量值的三分之一,不扣掉会把 CPU 成本显著高估。 +do_baseline() { + local secs=${1:-60} a=0 b=0 + echo "测量空闲基线 ${secs}s —— 请确保这段时间内没有任何负载 ..." + for s in $SQUIDS; do a=$(awk -v x="$a" -v y="$(cpu_usec "$s")" 'BEGIN{print x+y}'); done + sleep "$secs" + for s in $SQUIDS; do b=$(awk -v x="$b" -v y="$(cpu_usec "$s")" 'BEGIN{print x+y}'); done + awk -v a="$a" -v b="$b" -v t="$secs" 'BEGIN{printf "%.6f\n", (b-a)/1000000/t}' > "$BASELINE_FILE" + echo "空闲基线: $(cat "$BASELINE_FILE") CPU秒/秒 (三副本合计) → $BASELINE_FILE" } # 汇总区间内的 access.log。$1=起始行(0 表示全部) @@ -123,19 +139,37 @@ report() { fi echo "" - echo "── SSL Bump 层: CPU 成本 ──" + echo "── SSL Bump 层: CPU 成本 (两项模型) ──" awk -v b="$bumped" -v sp="$splice" -v sb="$sbytes" -v t="$tunnel" 'BEGIN{ printf " 隧道 %d 条: 解密(bump) %d / 直通(splice) %d", t, b, sp if (sp > 0) printf " 直通字节 %.2f GB(不可能命中)", sb/1073741824 printf "\n" }' - awk -v c="$cpu_s" -v b="$bytes" 'BEGIN{ - printf " CPU 合计 %.2f 秒 (三副本)\n", c - if (b > 0) printf " 每 GB CPU %.2f 秒/GB ← 容量推导的关键系数\n", c/(b/1073741824) - else printf " 每 GB CPU n/a (无流量)\n" + local base_rate=0 + [ -f "$BASELINE_FILE" ] && base_rate=$(cat "$BASELINE_FILE" 2>/dev/null || echo 0) + awk -v c="$cpu_s" -v br="${base_rate:-0}" -v e="$elapsed" -v b="$bytes" -v r="$req" -v hasb="$([ -f "$BASELINE_FILE" ] && echo 1 || echo 0)" 'BEGIN{ + idle = br * e + attr = c - idle; if (attr < 0) attr = 0 + if (hasb) printf " CPU 原始 %.2f 秒 − 空闲基线 %.2f 秒 = 归因 %.2f 秒\n", c, idle, attr + else printf " CPU 原始 %.2f 秒 (未扣空闲基线,偏高; 先跑一次 baseline)\n", c + if (r > 0) { + printf " 平均对象 %.2f MB\n", (b/r)/1048576 + printf " 每请求 CPU %.4f 秒/请求 ← 固定成本: TLS 握手 + 证书生成 + 查找\n", attr/r + } + if (b > 0) + printf " 每 GB CPU %.2f 秒/GB ← 边际成本: 加密 + I/O\n", attr/(b/1073741824) }' - echo " 注意: 纯 HIT 场景 CPU 高于纯 MISS(命中要重新做 TLS 加密,回源只是等 I/O)。" - echo " 要拿到可用的系数,请分别跑「纯 HIT」与「纯 MISS」两个窗口再比较。" + cat <<'NOTE' + ⚠ 「每 GB CPU」不是常数,它随对象大小变化 —— CPU 由「每请求固定成本」和 + 「每字节边际成本」两项构成,对象越小,固定成本被摊到的字节越少,该值越高。 + 实测同一套环境: 14MB 对象约 28 秒/GB,58KB-6MB 混合约 37-44 秒/GB。 + 容量估算请用两项模型: + 核数 ≈ 请求速率 × 每请求CPU + 吞吐(GB/s) × 每GB CPU + 要拟合这两项,需要在不同对象大小/并发下各跑一个窗口,再比较。 + ⚠ 低并发下测不出「HIT 比 MISS 更吃 CPU」—— 那是 N=120 量级 TLS 批量加密打满 + CPU 时才出现的效应(见 reports/stress-benchmark-20260811.md)。低并发时开销 + 主要在握手与证书生成,不要用小样本去推翻高并发结论。 +NOTE echo "" echo "── HAProxy 层: 分发与健康 ──" @@ -155,8 +189,9 @@ report() { } case "${1:-now}" in - begin) do_begin ;; - end) [ -f "$STATE" ] || { echo "没有基线,先跑 ./scripts/metrics.sh begin"; exit 1; }; report delta ;; - now) report full ;; - *) echo "用法: $0 {begin|end|now}"; exit 1 ;; + baseline) do_baseline "${2:-60}" ;; + begin) do_begin ;; + end) [ -f "$STATE" ] || { echo "没有基线,先跑 ./scripts/metrics.sh begin"; exit 1; }; report delta ;; + now) report full ;; + *) echo "用法: $0 {baseline [秒]|begin|end|now}"; exit 1 ;; esac From 17e66738f371db85c533c5a65939e6035023e90b Mon Sep 17 00:00:00 2001 From: Aron Xu Date: Tue, 11 Aug 2026 19:03:06 +0800 Subject: [PATCH 03/22] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=BB=B6=E8=BF=9F?= =?UTF-8?q?=E5=88=86=E4=BD=8D=E4=B8=8E=E9=94=99=E8=AF=AF=E5=88=86=E7=B1=BB?= =?UTF-8?q?=E4=B8=A4=E5=B1=82=E6=8C=87=E6=A0=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两层都从已经在采集的 access.log 字段算出来,不需要新数据源。 延迟层(字段 $2 耗时ms),按 HIT/MISS 分开出 p50/p95/p99: - 均值会被长尾骗过去。实测同一个 14MB 对象的 TCP_HIT 耗时分布为 133/246/251/256/273/5106 ms —— 同样是命中,最慢的比最快的高 38 倍。 这种尾巴用两次 wall-clock 采样(用例 04 现在的做法)根本看不见。 - 按命中结果分开是必要的: MISS 延迟基本由源站带宽支配,HIT 延迟才是 缓存真正买到的东西,混在一起两者互相污染。 - 对在线业务而言 p99 才是用户实际感受,这是从「省了多少带宽」转向 「用户等多久」时最先需要的指标。 错误层,关键是把代理故障与源站故障分开: - 判据是层级字段: Squid 自己生成的错误是 HIER_NONE(压根没连上源站), 真正到达源站再返回的错误带 HIER_DIRECT 等。 - 为什么重要: 源站 404(包不存在)是正常现象,不该告警;而 DNS 失败、 连接失败、ACL 拒绝才是要告警的。k8s 压测里 120 并发中 31-37 个「失败」 其实全是源站 404,不是 Squid 故障(见 reports/stress-benchmark-20260811.md)。 - 另外单列客户端中断(*_ABORTED)与隧道失败(CONNECT 阶段即失败)。 验证: 故意混入一个不存在的包,输出正确归类为「源站4xx 1 / 代理故障 0」。 实现上把窗口日志先落到临时文件再多趟分析,避免为每个指标重复 docker exec。 注意分位数的口径可以搬到生产,代码不能: Prometheus 要用预先分桶的 histogram,无法从计数器抓取值反算真实 p99。已在脚本头部注明。 Co-Authored-By: Claude Opus 5 (1M context) --- scripts/metrics.sh | 88 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/scripts/metrics.sh b/scripts/metrics.sh index 7ff1b20..1602107 100755 --- a/scripts/metrics.sh +++ b/scripts/metrics.sh @@ -1,8 +1,8 @@ #!/bin/bash -# 分层性能指标采集 —— 只依赖两个已有数据源,不引入任何新组件: -# 1) 各 Squid 的 access.log (命中/字节/客户端/bump-splice) +# 分层性能指标采集 —— 只依赖已有数据源,不引入任何新组件: +# 1) 各 Squid 的 access.log (命中/字节/客户端/延迟/错误分类/bump-splice) # 2) HAProxy stats CSV (:8404) (后端分发/健康状态) -# 3) cgroup cpu.stat (CPU 秒,用于算每 GB 的 CPU 成本) +# 3) cgroup cpu.stat (CPU 秒,用于算两项 CPU 成本) # # 用法: # ./scripts/metrics.sh baseline [秒] # 测空闲 CPU 基线(零负载时跑一次,可复用) @@ -13,6 +13,11 @@ # # 为什么要按区间而不是看累计: 累计值会把预热、健康检查、历史负载混在一起, # 得出的命中率和 CPU 成本都没有意义。必须框定一个已知负载的窗口。 +# +# 注意: 本脚本按窗口精确计算,适合测试与基准。生产用 Prometheus 抓取时代码不能直接搬 +# —— 分位数必须预先分桶(histogram),无法从计数器抓取值反算。可以搬的是这里的 +# **口径定义**(什么算一个请求、什么算命中、哪些行要排除),口径错了 Grafana 面板 +# 会和这里一样静默错掉。 set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" ROOT="$(dirname "$SCRIPT_DIR")" @@ -21,6 +26,9 @@ STATE="$ROOT/.metrics-state" BASELINE_FILE="$ROOT/.metrics-baseline" SQUIDS="squid1 squid2 squid3" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + # access.log 字段(squid 格式): # $1 时间戳 $2 耗时ms $3 客户端 $4 结果码/状态 $5 字节 # $6 方法 $7 URL $8 ident $9 层级/对端 $10 类型 @@ -37,6 +45,7 @@ $6 == "-" { next } # HAProxy 健康检查 # splice: TCP_TUNNEL/... 带真实字节 —— 未解密直通,Squid 看不到内容,不可能命中。 $6 == "CONNECT" { tunnel++ + split($4, t, "/"); if (t[2]+0 >= 400) tunfail++ if ($4 ~ /^TCP_TUNNEL/) { splice++; sbytes += $5 } else { bumped++ } next @@ -48,10 +57,23 @@ $6 == "CONNECT" { if ($4 ~ /TCP_(MEM_)?HIT/) { hit++; hitb += $5 } else if ($4 ~ /TCP_(REFRESH|IMS)_HIT/) { rhit++; hitb += $5 } else { miss++; missb += $5 } + + # 错误分类 —— 关键是把「代理故障」和「源站故障」分开。 + # 判据: Squid 自己生成的错误层级是 HIER_NONE(没真正连上源站), + # 真正到达源站再返回的错误层级是 HIER_DIRECT/HIER_*_PARENT 等。 + # 为什么重要: 源站 404(比如包不存在)是正常现象,不该告警; + # k8s 压测里 120 并发有 31-37 个「失败」其实全是源站 404,不是 Squid 故障。 + split($4, a, "/"); st = a[2] + 0 + if ($4 ~ /ABORTED/) aborted++ # 客户端中途断开 + else if (st < 400) ok++ + else if ($9 ~ /^HIER_NONE/) proxyerr++ # DNS/连接失败/拒绝 —— 该告警的是这个 + else if (st >= 500) origin5++ + else origin4++ } END { - printf "req=%d hit=%d rhit=%d miss=%d bytes=%d hitb=%d missb=%d bumped=%d splice=%d sbytes=%d tunnel=%d\n", - req, hit, rhit, miss, bytes, hitb, missb, bumped, splice, sbytes, tunnel + printf "req=%d hit=%d rhit=%d miss=%d bytes=%d hitb=%d missb=%d bumped=%d splice=%d sbytes=%d tunnel=%d tunfail=%d ok=%d origin4=%d origin5=%d proxyerr=%d aborted=%d\n", + req, hit, rhit, miss, bytes, hitb, missb, bumped, splice, sbytes, tunnel, tunfail, + ok, origin4, origin5, proxyerr, aborted for (c in cli) printf "CLIENT %s %d %d\n", c, clir[c], cli[c] }' @@ -86,22 +108,35 @@ do_baseline() { echo "空闲基线: $(cat "$BASELINE_FILE") CPU秒/秒 (三副本合计) → $BASELINE_FILE" } -# 汇总区间内的 access.log。$1=起始行(0 表示全部) -collect() { +# 把窗口内的 access.log 落到一个文件,后续多趟分析都读它(避免重复 docker exec) +fetch_window() { local mode="$1" + : > "$WORK/log" for s in $SQUIDS; do local from=0 [ "$mode" = "delta" ] && from=$(awk -v s="$s" '$1==s{print $2}' "$STATE" 2>/dev/null) from=${from:-0} - docker exec "$s" sh -c "tail -n +$((from + 1)) /var/log/squid/access.log" 2>/dev/null - done | awk "$AWK_AGG" + docker exec "$s" sh -c "tail -n +$((from + 1)) /var/log/squid/access.log" 2>/dev/null >> "$WORK/log" + done +} + +# 最近秩分位。$1=数字文件 $2=百分位 +pct() { + local f="$1" p="$2" n + n=$(wc -l < "$f" 2>/dev/null || echo 0); n=${n:-0} + [ "$n" -eq 0 ] && { echo "n/a"; return; } + sort -n "$f" | awk -v p="$p" -v n="$n" 'BEGIN{i=int(p*n/100); if(i<1)i=1} NR==i{printf "%d", $1; exit}' } report() { - local mode="$1" out - out=$(collect "$mode") - local agg; agg=$(echo "$out" | grep -v '^CLIENT ') - eval "$(echo "$agg" | tr ' ' '\n' | sed 's/^/M_/')" 2>/dev/null + local mode="$1" + fetch_window "$mode" + local out; out=$(awk "$AWK_AGG" "$WORK/log") + eval "$(echo "$out" | grep -v '^CLIENT ' | tr ' ' '\n' | sed 's/^/M_/')" 2>/dev/null + + # 延迟样本按命中结果分流(只取对象请求,排除健康检查与 CONNECT) + awk '$6 != "-" && $6 != "CONNECT" && $4 ~ /TCP_(MEM_)?HIT/ {print $2}' "$WORK/log" > "$WORK/lat_hit" + awk '$6 != "-" && $6 != "CONNECT" && $4 !~ /TCP_(MEM_|REFRESH_|IMS_)?HIT/ {print $2}' "$WORK/log" > "$WORK/lat_miss" local cpu_s=0 elapsed=0 for s in $SQUIDS; do @@ -117,7 +152,10 @@ report() { local req=${M_req:-0} hit=${M_hit:-0} rhit=${M_rhit:-0} miss=${M_miss:-0} local bytes=${M_bytes:-0} hitb=${M_hitb:-0} missb=${M_missb:-0} - local bumped=${M_bumped:-0} splice=${M_splice:-0} sbytes=${M_sbytes:-0} tunnel=${M_tunnel:-0} + local bumped=${M_bumped:-0} splice=${M_splice:-0} sbytes=${M_sbytes:-0} + local tunnel=${M_tunnel:-0} tunfail=${M_tunfail:-0} + local ok=${M_ok:-0} origin4=${M_origin4:-0} origin5=${M_origin5:-0} + local proxyerr=${M_proxyerr:-0} aborted=${M_aborted:-0} echo "============================================" echo " 分层性能指标 ($([ "$mode" = delta ] && echo "区间 ${elapsed}s" || echo "累计"))" @@ -132,12 +170,28 @@ report() { printf " 请求数 %d (HIT %d / REFRESH_HIT %d / MISS %d)\n", r, h, rh, m printf " 请求命中率 %.1f%%\n", (h+rh)*100/r printf " 字节命中率 %.1f%% ← 省下的出向带宽看这个\n", (b>0? hb*100/b : 0) - printf " 服务字节 %.2f GB\n", b/1073741824 - printf " 回源字节 %.2f GB\n", mb/1073741824 - printf " 省下回源 %.2f GB\n", hb/1073741824 + printf " 服务字节 %.2f GB 回源 %.2f GB 省下 %.2f GB\n", + b/1073741824, mb/1073741824, hb/1073741824 }' fi + echo "" + echo "── 延迟层: 按命中结果分位 (毫秒) ──" + printf " %-6s n=%-6s p50=%-8s p95=%-8s p99=%s\n" "HIT" \ + "$(wc -l < "$WORK/lat_hit" | tr -d ' ')" "$(pct "$WORK/lat_hit" 50)" "$(pct "$WORK/lat_hit" 95)" "$(pct "$WORK/lat_hit" 99)" + printf " %-6s n=%-6s p50=%-8s p95=%-8s p99=%s\n" "MISS" \ + "$(wc -l < "$WORK/lat_miss" | tr -d ' ')" "$(pct "$WORK/lat_miss" 50)" "$(pct "$WORK/lat_miss" 95)" "$(pct "$WORK/lat_miss" 99)" + echo " MISS 延迟 ≈ 源站延迟 + 代理开销(受源站带宽支配); HIT 延迟才是缓存真正买到的东西。" + echo " 看均值会被长尾骗过去 —— 对在线业务 p99 才是用户实际感受。" + + echo "" + echo "── 错误层: 代理故障 vs 源站故障 ──" + printf " 成功(2xx/3xx) %-6s 源站4xx %-6s 源站5xx %-6s 代理故障 %-6s 客户端中断 %s\n" \ + "$ok" "$origin4" "$origin5" "$proxyerr" "$aborted" + [ "$tunfail" -gt 0 ] && echo " 隧道失败 $tunfail (CONNECT 阶段即失败)" + echo " 源站 4xx 多半是包不存在等正常现象,不应告警;" + echo " 「代理故障」才是要告警的(HIER_NONE 且 >=400: DNS/连接失败/ACL 拒绝)。" + echo "" echo "── SSL Bump 层: CPU 成本 (两项模型) ──" awk -v b="$bumped" -v sp="$splice" -v sb="$sbytes" -v t="$tunnel" 'BEGIN{ From 91a15b6531bf8ea212085fa2c61ff63b9c2b64ea Mon Sep 17 00:00:00 2001 From: TommyLike Date: Wed, 12 Aug 2026 17:42:54 +0800 Subject: [PATCH 04/22] =?UTF-8?q?test(04):=20=E6=94=B9=E7=94=A8=E5=88=86?= =?UTF-8?q?=E5=B1=82=E6=8C=87=E6=A0=87(=E5=BB=B6=E8=BF=9F=E5=88=86?= =?UTF-8?q?=E4=BD=8D+CPU=E4=B8=A4=E9=A1=B9=E6=88=90=E6=9C=AC+=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E5=88=86=E7=B1=BB)=E5=8F=96=E4=BB=A3=E4=B8=A4?= =?UTF-8?q?=E6=AC=A1=E9=87=87=E6=A0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 依据 PR #12(scripts/metrics.sh)的建议改造用例 04。旧版只打印首次(MISS)/ 二次(HIT)两个孤立 wall-clock,看不见长尾、不分类错误、不测 CPU 成本。 新版用 metrics.sh begin/end 框定 MISS 与 HIT 两个窗口: - 固定客户端 IP(balance source → 同一台 Squid)+ cache-busting URL, 保证窗口1是真 MISS、窗口2是真 HIT(各 Squid 缓存独立、无 peer) - 先测空闲 CPU 基线并在归因时扣除 - 保留 HTTP 200 真断言,分层指标作为信息输出 实测效果(本地 Compose): - MISS p50=3638ms / 命中率0% / 每请求CPU 0.247s - HIT p50=130 p95=148 p99=149ms / 命中率100% / 每请求CPU 0.085s - 缓存价值 = 延迟 28× 提速; CONNECT 隧道正确去重(未重复计数) - 归因层正确检测到 Compose 侧无 PROXY protocol(#9)并提示 README 同步更新测试点表与预期输出。 Co-Authored-By: Claude Opus 4.8 --- README.md | 16 +++++------ tests/04-https-cache.sh | 62 +++++++++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 44b2ecc..bcd0f9c 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ git clone https://github.com/opensourceways/squid_e2e_tests.git && cd squid_e2e_ | 01 | HTTP + HTTPS 代理连通(HTTP 200 断言) | | 02 | 单 Squid 故障,代理仍返回 200 | | 03 | HAProxy 节点故障,VIP 双向漂移,代理仍返回 200 | -| 04 | HTTPS SSL Bump 缓存,展示首次(MISS)与二次(HIT)耗时对比 | +| 04 | HTTPS SSL Bump 缓存,**分层指标**: HIT/MISS 延迟分位(p50/p95/p99)+ CPU 两项成本 + 错误分类(用 `scripts/metrics.sh`) | | 05 | 下载中 Squid 中断,代理仍返回 200 | | 06 | 两台 Squid 同时故障(仅剩 1/3),代理仍返回 200 | @@ -100,13 +100,13 @@ git clone https://github.com/opensourceways/squid_e2e_tests.git && cd squid_e2e_ ✓ 恢复后代理: HTTP 200 [02-squid-failover.sh] PASS (24s) ... ---- 04-https-cache.sh --- - ✓ 首次下载(MISS): HTTP 200 - 首次(MISS): 2.6s 15000000 bytes 5700000 B/s - ✓ 二次下载(HIT): HTTP 200 - 二次(HIT): 0.16s 15000000 bytes 95000000 B/s - 耗时对比: 首次 2.6s → 二次 0.16s - [04-https-cache.sh] PASS (5s) +--- 04-https-cache.sh --- (分层指标, 用 metrics.sh 框定 MISS/HIT 两个窗口) + ✓ 冷取(MISS): HTTP 200 + MISS n=1 p50=3638ms 请求命中率 0% 每请求CPU 0.247s 每GB 17.8s + ✓ 命中(HIT): HTTP 200 + HIT n=30 p50=130 p95=148 p99=149 ms 请求命中率 100% 每请求CPU 0.085s 每GB 6.1s + → 缓存价值 = 延迟 28× 提速(3638→130ms); 两次采样看不到 p95/p99 分布与 CPU 成本 + [04-https-cache.sh] PASS ... ============================================ 结果: 6 通过 / 0 失败 / 6 总计 diff --git a/tests/04-https-cache.sh b/tests/04-https-cache.sh index a57fa1e..1d4ede2 100755 --- a/tests/04-https-cache.sh +++ b/tests/04-https-cache.sh @@ -1,26 +1,58 @@ #!/bin/bash -# 04: HTTPS SSL Bump 缓存 —— 显示首次(MISS)与二次(HIT)耗时对比 +# 04: HTTPS SSL Bump 缓存 —— 分层指标(命中延迟分位 + CPU 两项成本 + 错误分类) +# +# 取代旧的"两次 wall-clock 采样": 两次采样看不见长尾、不分类错误、不测 CPU 成本。 +# 依据 scripts/metrics.sh(PR #12)的口径, 用 begin/end 框定 MISS 窗口与 HIT 窗口。 set -e source "$(dirname "$0")/lib.sh" -echo "=== 04: HTTPS SSL Bump 缓存 ===" +METRICS="$(dirname "$0")/../scripts/metrics.sh" +echo "=== 04: HTTPS SSL Bump 缓存(分层指标) ===" echo "URL: $HTTPS_CACHE_URL" -# 返回 "http_code time_total size_download speed_download" -https_dl() { - docker run --rm --network "$NET" -v "$CA:/tmp/ca.crt:ro" alpine/curl:latest \ - curl --cacert /tmp/ca.crt --connect-timeout 10 -s -o /dev/null \ - -w "%{http_code} %{time_total} %{size_download} %{speed_download}" \ - -x "http://$VIP" "$HTTPS_CACHE_URL" --max-time 300 2>/dev/null +# 固定客户端 IP: balance source 按源 IP 哈希 → 固定落到同一台 Squid。各 Squid 缓存独立、 +# 无 peer, 所以 prime 与 HIT 负载必须同一 IP 才命中同一份缓存(否则 HIT 窗口会变 MISS)。 +CLIENT_IP=172.30.0.50 + +# cache-busting: 附一个唯一 query 串, 让 Squid 视作从未见过的新对象。 +# (strip_query_terms 只影响日志显示, 不影响缓存 key → 保证窗口1是真 MISS) +# 两个窗口用同一个 busted URL: 窗口1 冷取=MISS, 窗口2 复取=HIT。 +BUSTED_URL="${HTTPS_CACHE_URL}?cb=$(date +%s)-$$-${RANDOM}" + +# 用固定 IP 的一个容器连打 N 次(容器启动开销只付一次), 返回首个 http_code。 +hammer() { + docker run --rm --network "$NET" --ip "$CLIENT_IP" -v "$CA:/tmp/ca.crt:ro" \ + -e N="$1" -e VIP="$VIP" -e URL="$BUSTED_URL" \ + --entrypoint sh alpine/curl:latest -c ' + code=""; i=0 + while [ "$i" -lt "$N" ]; do + c=$(curl --cacert /tmp/ca.crt -s -o /dev/null -w "%{http_code}" \ + -x "http://$VIP" "$URL" --max-time 300) + [ -z "$code" ] && code=$c + i=$((i+1)) + done + echo "$code"' 2>/dev/null } -read -r C1 T1 S1 SP1 <<< "$(https_dl)" -assert_code "首次下载(MISS)" 200 "$C1" -printf " 首次(MISS): %ss %s bytes %.0f B/s\n" "$T1" "$S1" "$SP1" +# 0) 空闲 CPU 基线(此刻环境应无负载)—— end 会按窗口时长扣除, 避免把空闲 CPU 算进请求成本 +echo "" +echo "--- 测空闲 CPU 基线(15s, 请勿在此期间加负载) ---" +"$METRICS" baseline 15 >/dev/null -read -r C2 T2 S2 SP2 <<< "$(https_dl)" -assert_code "二次下载(HIT)" 200 "$C2" -printf " 二次(HIT): %ss %s bytes %.0f B/s\n" "$T2" "$S2" "$SP2" +# 1) MISS 窗口: 冷取一次(把对象灌进 CLIENT_IP 对应那台 Squid 的缓存) +echo "" +echo "--- MISS 窗口: 冷取 1 次 ---" +"$METRICS" begin >/dev/null +CM=$(hammer 1) +assert_code "冷取(MISS)" 200 "$CM" +"$METRICS" end -echo " 耗时对比: 首次 ${T1}s → 二次 ${T2}s" +# 2) HIT 窗口: 同一 IP 命中同一份缓存, 连打 30 次看 HIT 延迟分布(p50/p95/p99) +echo "" +echo "--- HIT 窗口: 命中 30 次(同一缓存) ---" +"$METRICS" begin >/dev/null +CH=$(hammer 30) +assert_code "命中(HIT)" 200 "$CH" +"$METRICS" end +echo "" echo "PASS" From 1b771ac22a0a81d46fd2e42804d38564b913d1aa Mon Sep 17 00:00:00 2001 From: TommyLike Date: Wed, 12 Aug 2026 17:58:26 +0800 Subject: [PATCH 05/22] =?UTF-8?q?test(k8s):=20=E5=88=86=E5=B1=82=E6=8C=87?= =?UTF-8?q?=E6=A0=87=E5=90=8C=E6=AC=BE=E9=87=87=E9=9B=86(metrics-k8s.sh=20?= =?UTF-8?q?+=20latency-test.sh),=E5=B7=B2=E5=9C=A8=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E8=B7=91=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 把 tests/04 的分层指标口径搬到 K8s(与 Compose 版 metrics.sh 完全同口径: 健康检查过滤 / CONNECT 去重 / HIER 错误分类 / HIT-MISS 延迟分位 / CPU 两项成本), 只把数据源从 docker+HAProxy 换成 kubectl exec(access.log、cgroup v1 cpuacct.usage) + kubectl get endpoints。 - k8s/metrics-k8s.sh: K8s 版采集(baseline/begin/end/now) - k8s/latency-test.sh: 一键 driver(起 client pod → baseline → MISS/HIT 两窗口 → 清理), 等价 Compose 的 tests/04 - .gitignore: 忽略 .metrics-k8s-{state,baseline} - README: 新增「K8s 上的同款采集」小节 测试集群(test-husheng, squid StatefulSet 2/3 Ready)实测: - MISS p50≈1000ms / 命中率0% ; HIT n=30 p50≈167 p95≈176 p99≈177ms / 命中率100% - CONNECT 隧道正确去重(bump 1 / bump 30) - HIT 每请求 CPU(0.177)略高于 MISS(0.140),方向与「HIT 重新加密更吃 CPU」一致 - 归因层直接显示真实客户端 pod 段 IP —— K8s Service 保留源 IP,无需 PROXY protocol Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 + README.md | 16 +++++ k8s/latency-test.sh | 71 ++++++++++++++++++ k8s/metrics-k8s.sh | 170 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 259 insertions(+) create mode 100755 k8s/latency-test.sh create mode 100755 k8s/metrics-k8s.sh diff --git a/.gitignore b/.gitignore index b5c7f1a..5fa35c4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ k8s/*.csv buildkit/src/ .metrics-state .metrics-baseline +.metrics-k8s-state +.metrics-k8s-baseline diff --git a/README.md b/README.md index bcd0f9c..658b757 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,22 @@ CPU 消耗 = **每请求固定成本**(TLS 握手、证书生成、缓存查找) 日志写入、缓存索引维护)。窗口越稀疏——比如里面有大量 `docker run` 启动等待——这部分占比越高, 实测能占到测量值的三分之一。不扣掉会显著高估 CPU 成本,也会让两个窗口之间的比较失去意义。 +### K8s 上的同款采集 + +`k8s/metrics-k8s.sh` 是 metrics.sh 的 K8s 版——**口径完全相同**(健康检查过滤、CONNECT 去重、 +HIER 错误分类、HIT/MISS 延迟分位、CPU 两项成本),只把数据源从 docker+HAProxy 换成 +`kubectl exec`(access.log、cgroup **v1** `cpuacct.usage`)+ `kubectl get endpoints`。 +一键跑 MISS/HIT 两窗口: + +```bash +NS=test-husheng ./k8s/latency-test.sh # 起 client pod → baseline → MISS 窗口 → HIT 窗口 → 清理 +``` + +两点与 Compose 侧不同:① 没有 HAProxy/VIP,负载均衡是 Service `sessionAffinity: ClientIP`; +② **K8s Service 保留客户端源 IP,归因层无需 PROXY protocol 即可用**(Compose 侧 Squid 只看得到 +HAProxy 节点 IP)。实测 HIT p50≈167ms、MISS p50≈1000ms,HIT 每请求 CPU 略高于 MISS +——与"HIT 需重新加密更吃 CPU"方向一致。 + ## 其他部分 | 部分 | 位置 | 说明 | diff --git a/k8s/latency-test.sh b/k8s/latency-test.sh new file mode 100755 index 0000000..e39e382 --- /dev/null +++ b/k8s/latency-test.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# K8s 版"用例 04": 分层指标(HIT/MISS 延迟分位 + CPU 两项成本 + 错误分类)。 +# 等价于 tests/04-https-cache.sh, 但跑在 K8s 的 squid StatefulSet + Service 上, +# 采集用 k8s/metrics-k8s.sh(同口径, 数据源换成 kubectl exec + endpoints)。 +# +# 用法: NS=test-husheng ./k8s/latency-test.sh [RPM_URL] +# 前置: squid StatefulSet + Service(squid:3128, ssl-bump)已部署、squid-ca secret 存在。 +set -e +NS="${NS:-test-husheng}" +K8S_DIR="$(cd "$(dirname "$0")" && pwd)" +METRICS="$K8S_DIR/metrics-k8s.sh" +PROXY="http://squid.${NS}.svc.cluster.local:3128" +BASE_URL="${1:-https://repo.openeuler.org/openEuler-23.03/debuginfo/aarch64/Packages/bcc-debuginfo-0.26.0-1.oe2303.aarch64.rpm}" +# cache-busting: 唯一 query 串 → 保证窗口1是真 MISS(strip_query_terms 只影响日志, 不影响缓存 key) +URL="${BASE_URL}?cb=$(date +%s)-${RANDOM}" +POD=metrics-client + +cleanup() { kubectl -n "$NS" delete pod "$POD" --grace-period=0 --force >/dev/null 2>&1 || true; } +trap cleanup EXIT +cleanup + +echo "=== K8s 分层指标测试 ns=$NS ===" +echo "URL: $BASE_URL" + +# 固定身份的 client pod: Service sessionAffinity=ClientIP → 固定落到同一台 squid, +# prime 与 HIT 命中同一份缓存(各副本 PVC 独立)。挂 squid-ca 做 SSL-Bump 信任。 +kubectl -n "$NS" apply -f - >/dev/null </dev/null + +# 在 client pod 内连打 N 次, 打印首个 http_code +hammer() { + kubectl -n "$NS" exec "$POD" -- sh -c ' + code=""; i=0 + while [ $i -lt '"$1"' ]; do + c=$(curl --cacert /etc/ca/squid-ca.pem -s -o /dev/null -w "%{http_code}" -x "'"$PROXY"'" "'"$URL"'" --max-time 300) + [ -z "$code" ] && code=$c; i=$((i+1)) + done; echo "$code"' +} + +echo ""; echo "--- 空闲 CPU 基线 15s ---" +NS="$NS" "$METRICS" baseline 15 + +echo ""; echo "--- MISS 窗口: 冷取 1 次 ---" +NS="$NS" "$METRICS" begin >/dev/null +CM=$(hammer 1) +[ "$CM" = 200 ] || { echo "✗ FAIL 冷取 http_code=$CM"; exit 1; } +echo "✓ 冷取(MISS): HTTP $CM" +NS="$NS" "$METRICS" end + +echo ""; echo "--- HIT 窗口: 命中 30 次 ---" +NS="$NS" "$METRICS" begin >/dev/null +CH=$(hammer 30) +[ "$CH" = 200 ] || { echo "✗ FAIL 命中 http_code=$CH"; exit 1; } +echo "✓ 命中(HIT): HTTP $CH" +NS="$NS" "$METRICS" end + +echo ""; echo "PASS" diff --git a/k8s/metrics-k8s.sh b/k8s/metrics-k8s.sh new file mode 100755 index 0000000..75b8ba1 --- /dev/null +++ b/k8s/metrics-k8s.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# 分层性能指标(K8s 版)—— 与 scripts/metrics.sh 同口径, 数据源换成 K8s: +# 1) 各 squid pod 的 access.log (kubectl exec ... cat) 命中/字节/客户端/延迟/错误/bump-splice +# 2) cgroup v1 cpuacct.usage (kubectl exec) CPU 秒, 用于两项 CPU 成本 +# 3) endpoints / ready pods 取代 Compose 侧的 HAProxy stats +# +# 与 Compose 版的差异: +# - 没有 HAProxy/VIP。负载均衡是 Service(sessionAffinity: ClientIP, 等价 balance source)。 +# - cgroup 是 v1(cpuacct.usage 纳秒), 不是 v2 的 cpu.stat usage_usec。 +# - K8s Service 不像 HAProxy TCP 模式那样隐藏源 IP, 归因层通常直接可用(显示真实客户端 pod IP)。 +# +# 用法: +# NS=test-husheng ./k8s/metrics-k8s.sh baseline [秒] +# NS=test-husheng ./k8s/metrics-k8s.sh begin +# <跑负载> +# NS=test-husheng ./k8s/metrics-k8s.sh end +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" +NS="${NS:-test-husheng}" +STATE="$ROOT/.metrics-k8s-state" +BASELINE_FILE="$ROOT/.metrics-k8s-baseline" + +WORK=$(mktemp -d); trap 'rm -rf "$WORK"' EXIT + +squids() { kubectl -n "$NS" get pods -l app=squid-cache --field-selector=status.phase=Running \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null; } +SQUIDS=$(squids) + +# 与 scripts/metrics.sh 完全相同的口径(健康检查剔除 / CONNECT 隧道去重 / HIER 错误分类) +AWK_AGG=' +$6 == "-" { next } +{ cli[$3] += $5; clir[$3]++ } +$6 == "CONNECT" { + tunnel++; split($4, t, "/"); if (t[2]+0 >= 400) tunfail++ + if ($4 ~ /^TCP_TUNNEL/) { splice++; sbytes += $5 } else { bumped++ } + next +} +{ + req++; bytes += $5 + if ($4 ~ /TCP_(MEM_)?HIT/) { hit++; hitb += $5 } + else if ($4 ~ /TCP_(REFRESH|IMS)_HIT/) { rhit++; hitb += $5 } + else { miss++; missb += $5 } + split($4, a, "/"); st = a[2] + 0 + if ($4 ~ /ABORTED/) aborted++ + else if (st < 400) ok++ + else if ($9 ~ /^HIER_NONE/) proxyerr++ + else if (st >= 500) origin5++ + else origin4++ +} +END { + printf "req=%d hit=%d rhit=%d miss=%d bytes=%d hitb=%d missb=%d bumped=%d splice=%d sbytes=%d tunnel=%d tunfail=%d ok=%d origin4=%d origin5=%d proxyerr=%d aborted=%d\n", + req,hit,rhit,miss,bytes,hitb,missb,bumped,splice,sbytes,tunnel,tunfail,ok,origin4,origin5,proxyerr,aborted + for (c in cli) printf "CLIENT %s %d %d\n", c, clir[c], cli[c] +}' + +# cgroup v1: cpuacct.usage 是纳秒累计 +cpu_ns() { kubectl -n "$NS" exec "$1" -- cat /sys/fs/cgroup/cpuacct/cpuacct.usage 2>/dev/null || echo 0; } +log_lines() { kubectl -n "$NS" exec "$1" -- sh -c 'wc -l < /var/log/squid/access.log' 2>/dev/null | tr -d ' ' || echo 0; } + +do_begin() { + : > "$STATE" + for s in $SQUIDS; do echo "$s $(log_lines "$s") $(cpu_ns "$s")" >> "$STATE"; done + echo "TS $(date +%s)" >> "$STATE" + echo "基线已记录 ($STATE)。跑完负载后: NS=$NS $0 end" + [ -f "$BASELINE_FILE" ] || echo "提示: 尚未测空闲基线, CPU 归因会偏高。先跑一次 baseline" +} + +do_baseline() { + local secs=${1:-30} a=0 b=0 + echo "测量空闲基线 ${secs}s (pods: $(echo $SQUIDS | tr '\n' ' ')) —— 请勿在此期间加负载 ..." + for s in $SQUIDS; do a=$(awk -v x="$a" -v y="$(cpu_ns "$s")" 'BEGIN{print x+y}'); done + sleep "$secs" + for s in $SQUIDS; do b=$(awk -v x="$b" -v y="$(cpu_ns "$s")" 'BEGIN{print x+y}'); done + awk -v a="$a" -v b="$b" -v t="$secs" 'BEGIN{printf "%.6f\n", (b-a)/1e9/t}' > "$BASELINE_FILE" + echo "空闲基线: $(cat "$BASELINE_FILE") CPU秒/秒 (合计 $(echo $SQUIDS | wc -w | tr -d ' ') 副本) → $BASELINE_FILE" +} + +fetch_window() { + local mode="$1"; : > "$WORK/log" + for s in $SQUIDS; do + local from=0 + [ "$mode" = "delta" ] && from=$(awk -v s="$s" '$1==s{print $2}' "$STATE" 2>/dev/null) + from=${from:-0} + kubectl -n "$NS" exec "$s" -- sh -c "tail -n +$((from + 1)) /var/log/squid/access.log" 2>/dev/null >> "$WORK/log" + done +} + +pct() { + local f="$1" p="$2" n; n=$(wc -l < "$f" 2>/dev/null || echo 0); n=${n:-0} + [ "$n" -eq 0 ] && { echo "n/a"; return; } + sort -n "$f" | awk -v p="$p" -v n="$n" 'BEGIN{i=int(p*n/100); if(i<1)i=1} NR==i{printf "%d",$1; exit}' +} + +report() { + local mode="$1"; fetch_window "$mode" + local out; out=$(awk "$AWK_AGG" "$WORK/log") + eval "$(echo "$out" | grep -v '^CLIENT ' | tr ' ' '\n' | sed 's/^/M_/')" 2>/dev/null + awk '$6 != "-" && $6 != "CONNECT" && $4 ~ /TCP_(MEM_)?HIT/ {print $2}' "$WORK/log" > "$WORK/lat_hit" + awk '$6 != "-" && $6 != "CONNECT" && $4 !~ /TCP_(MEM_|REFRESH_|IMS_)?HIT/ {print $2}' "$WORK/log" > "$WORK/lat_miss" + + local cpu_s=0 elapsed=0 + for s in $SQUIDS; do + local now b=0; now=$(cpu_ns "$s") + [ "$mode" = "delta" ] && b=$(awk -v s="$s" '$1==s{print $3}' "$STATE" 2>/dev/null) + cpu_s=$(awk -v a="$cpu_s" -v n="${now:-0}" -v bb="${b:-0}" 'BEGIN{printf "%.3f", a+(n-bb)/1e9}') + done + [ "$mode" = "delta" ] && { local t0; t0=$(awk '$1=="TS"{print $2}' "$STATE"); elapsed=$(( $(date +%s) - ${t0:-0} )); } + + local req=${M_req:-0} hit=${M_hit:-0} rhit=${M_rhit:-0} miss=${M_miss:-0} + local bytes=${M_bytes:-0} hitb=${M_hitb:-0} missb=${M_missb:-0} + local bumped=${M_bumped:-0} splice=${M_splice:-0} sbytes=${M_sbytes:-0} + local tunnel=${M_tunnel:-0} tunfail=${M_tunfail:-0} + local ok=${M_ok:-0} origin4=${M_origin4:-0} origin5=${M_origin5:-0} proxyerr=${M_proxyerr:-0} aborted=${M_aborted:-0} + + echo "============================================" + echo " 分层性能指标 · K8s ($([ "$mode" = delta ] && echo "区间 ${elapsed}s" || echo 累计)) ns=$NS" + echo "============================================" + echo "" + echo "── Squid 层: 缓存效果 (只统计可缓存对象请求) ──" + if [ "$req" -eq 0 ]; then echo " (区间内没有对象请求; 健康检查与 CONNECT 隧道已剔除)"; else + awk -v r="$req" -v h="$hit" -v rh="$rhit" -v m="$miss" -v b="$bytes" -v hb="$hitb" -v mb="$missb" 'BEGIN{ + printf " 请求数 %d (HIT %d / REFRESH_HIT %d / MISS %d)\n", r,h,rh,m + printf " 请求命中率 %.1f%%\n", (h+rh)*100/r + printf " 字节命中率 %.1f%% ← 省下的出向带宽看这个\n", (b>0? hb*100/b:0) + printf " 服务字节 %.2f GB 回源 %.2f GB 省下 %.2f GB\n", b/1073741824, mb/1073741824, hb/1073741824 }' + fi + echo "" + echo "── 延迟层: 按命中结果分位 (毫秒) ──" + printf " %-6s n=%-6s p50=%-8s p95=%-8s p99=%s\n" HIT \ + "$(wc -l < "$WORK/lat_hit" | tr -d ' ')" "$(pct "$WORK/lat_hit" 50)" "$(pct "$WORK/lat_hit" 95)" "$(pct "$WORK/lat_hit" 99)" + printf " %-6s n=%-6s p50=%-8s p95=%-8s p99=%s\n" MISS \ + "$(wc -l < "$WORK/lat_miss" | tr -d ' ')" "$(pct "$WORK/lat_miss" 50)" "$(pct "$WORK/lat_miss" 95)" "$(pct "$WORK/lat_miss" 99)" + echo " 看均值会被长尾骗过去 —— 对在线业务 p99 才是用户实际感受。" + echo "" + echo "── 错误层: 代理故障 vs 源站故障 ──" + printf " 成功(2xx/3xx) %-6s 源站4xx %-6s 源站5xx %-6s 代理故障 %-6s 客户端中断 %s\n" "$ok" "$origin4" "$origin5" "$proxyerr" "$aborted" + [ "$tunfail" -gt 0 ] && echo " 隧道失败 $tunfail" + echo " 「代理故障」才是要告警的(HIER_NONE 且 >=400: DNS/连接失败/ACL 拒绝); 源站4xx多为正常。" + echo "" + echo "── SSL Bump 层: CPU 成本 (两项模型) ──" + awk -v b="$bumped" -v sp="$splice" -v sb="$sbytes" -v t="$tunnel" 'BEGIN{ + printf " 隧道 %d 条: 解密(bump) %d / 直通(splice) %d", t,b,sp + if (sp>0) printf " 直通字节 %.2f GB", sb/1073741824; printf "\n" }' + local br=0; [ -f "$BASELINE_FILE" ] && br=$(cat "$BASELINE_FILE" 2>/dev/null || echo 0) + awk -v c="$cpu_s" -v br="${br:-0}" -v e="$elapsed" -v b="$bytes" -v r="$req" -v hasb="$([ -f "$BASELINE_FILE" ] && echo 1 || echo 0)" 'BEGIN{ + idle=br*e; attr=c-idle; if(attr<0)attr=0 + if(hasb) printf " CPU 原始 %.2f 秒 − 空闲基线 %.2f 秒 = 归因 %.2f 秒\n", c, idle, attr + else printf " CPU 原始 %.2f 秒 (未扣空闲基线, 偏高)\n", c + if(r>0){ printf " 平均对象 %.2f MB\n", (b/r)/1048576 + printf " 每请求 CPU %.4f 秒/请求 ← 固定成本: TLS握手+证书生成+查找\n", attr/r } + if(b>0) printf " 每 GB CPU %.2f 秒/GB ← 边际成本: 加密+I/O\n", attr/(b/1073741824) }' + echo " ⚠ 每GB CPU 非常数(随对象大小漂移); 低并发测不出「HIT比MISS更吃CPU」(见 reports/stress-benchmark-20260811.md)。" + echo "" + echo "── 后端层: 就绪副本 (取代 Compose 的 HAProxy stats) ──" + kubectl -n "$NS" get endpoints squid -o jsonpath='{range .subsets[*].addresses[*]} 就绪端点 {.ip}{"\n"}{end}' 2>/dev/null + kubectl -n "$NS" get pods -l app=squid-cache -o custom-columns=POD:.metadata.name,READY:.status.containerStatuses[0].ready,STATUS:.status.phase --no-headers 2>/dev/null | sed 's/^/ /' + echo "" + echo "── 归因层: 客户端分布 (K8s Service 保留源 IP, 无需 PROXY protocol) ──" + echo "$out" | grep '^CLIENT ' | sort -k4 -rn | head -8 | awk '{printf " %-16s 请求 %-6s 字节 %.2f MB\n", $2,$3,$4/1048576}' + echo "" +} + +case "${1:-now}" in + baseline) do_baseline "${2:-30}" ;; + begin) do_begin ;; + end) [ -f "$STATE" ] || { echo "先跑 begin"; exit 1; }; report delta ;; + now) report full ;; + *) echo "用法: NS=$NS $0 {baseline [秒]|begin|end|now}"; exit 1 ;; +esac From 15601392f2acdf2cc592a3d26d9cec936e19e4fe Mon Sep 17 00:00:00 2001 From: TommyLike Date: Thu, 13 Aug 2026 11:50:14 +0800 Subject: [PATCH 06/22] =?UTF-8?q?docs(buildkit):=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E4=B8=8A=E6=B8=B8=20PR=20#6996(env=20=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=BC=8F=E4=B8=8A=E6=B8=B8=E4=BB=A3=E7=90=86)=E7=9A=84?= =?UTF-8?q?=E7=AB=AF=E5=88=B0=E7=AB=AF=E9=AA=8C=E8=AF=81=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E4=B8=8E=E9=83=A8=E7=BD=B2=E8=B5=84=E4=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 从 gmarmstrong/buildkit@proxy-chaining 编译镜像, 正对照/缓存/负对照三场景实测通过 - 关键结论: 无 cert 配置 → CA 经 SSL_CERT_FILE 走系统信任库, 实测成立; PULL+RUN 共用一套 env 是 BuildKit 既有标准行为, 与 deploy 的 splice+registry-proxy 互补 - 新增 buildkitd-6996.toml / docker-compose.6996.yml, 与 fork 版并列 Co-Authored-By: Claude Opus 4.8 --- buildkit/README.md | 22 +++- buildkit/buildkitd-6996.toml | 14 ++ buildkit/docker-compose.6996.yml | 63 +++++++++ .../buildkit-pr6996-validation-20260813.md | 122 ++++++++++++++++++ 4 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 buildkit/buildkitd-6996.toml create mode 100644 buildkit/docker-compose.6996.yml create mode 100644 reports/buildkit-pr6996-validation-20260813.md diff --git a/buildkit/README.md b/buildkit/README.md index 813374c..151d09a 100644 --- a/buildkit/README.md +++ b/buildkit/README.md @@ -18,6 +18,22 @@ Squid HA,完成**代理 + SSL Bump 缓存**。 预构建镜像已推送,无需自行编译:`tommylike/buildkit-upstream-proxy:latest` (自行构建见 `image/README.md`) +## 上游 PR #6996 变体(env 变量式,已端到端验证) + +上游社区(gmarmstrong)在 moby/buildkit#6996 用**环境变量**实现了同一能力: +上游代理从 buildkitd 进程的 `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` 读取,**无 cert 配置**, +CA 信任走系统信任库(`SSL_CERT_FILE`)。与上面 fork 方案(`[proxy]` 段 + `upstreamCACert`) +的差异及端到端验证结果见 **`../reports/buildkit-pr6996-validation-20260813.md`**。 + +验证资产(与 fork 版并列,互不影响): + +- `buildkitd-6996.toml` —— 无 `[proxy]` 段,仅 `proxyNetwork = true` +- `docker-compose.6996.yml` —— env 变量式部署(`HTTP(S)_PROXY` + `SSL_CERT_FILE`) +- 镜像:`tommylike/buildkit-6996:test`(从 `gmarmstrong/buildkit@proxy-chaining` 编译) + +关键实测结论:正对照(RUN HTTPS 经 Squid 下载成功)、缓存命中(TCP_HIT 增加)、 +负对照(无 CA → 502)全部符合设计;一次 env 配置同时覆盖 **PULL + RUN** 两个场景。 + ## 快速开始 ```bash @@ -61,8 +77,10 @@ Squid HA (VIP) SSL Bump 解密 → 缓存 → 回源 buildkit/ ├── README.md ← 本文件 ├── DESIGN.md ← 设计与信任链、权限实证 -├── buildkitd.toml ← proxyNetwork + [proxy] upstreamURL/CACert -├── docker-compose.buildkit.yml ← buildkitd(精简 cap) +├── buildkitd.toml ← proxyNetwork + [proxy] upstreamURL/CACert(fork 版) +├── docker-compose.buildkit.yml ← buildkitd(精简 cap, fork 版) +├── buildkitd-6996.toml ← 上游 #6996 变体(env 变量式, 无 [proxy] 段) +├── docker-compose.6996.yml ← 上游 #6996 变体(HTTP(S)_PROXY + SSL_CERT_FILE) ├── Dockerfile.test ← 测试镜像(RUN curl https) ├── run-tests.sh ← 扩展测试入口(起停 buildkitd + 跑用例) ├── image/README.md ← 如何自行编译镜像 diff --git a/buildkit/buildkitd-6996.toml b/buildkit/buildkitd-6996.toml new file mode 100644 index 0000000..ec36e5e --- /dev/null +++ b/buildkit/buildkitd-6996.toml @@ -0,0 +1,14 @@ +# buildkitd 配置 —— 基于上游 PR #6996(env 变量式上游代理, 无 [proxy] 段) +# 上游代理完全由 buildkitd 进程环境变量决定: HTTP_PROXY / HTTPS_PROXY / NO_PROXY +# CA 信任走系统信任库: 容器里设 SSL_CERT_FILE=/etc/buildkit/squid-ca.pem + +# 开启内置 MITM 代理:给 RUN exec 注入 HTTP(S)_PROXY 并注入 CA +proxyNetwork = true + +[worker.oci] +enabled = true +# 沙箱网络用 host,确保 RUN 沙箱/代理出向能路由到 Squid VIP(172.30.0.100) +net = "host" + +[worker.containerd] +enabled = false diff --git a/buildkit/docker-compose.6996.yml b/buildkit/docker-compose.6996.yml new file mode 100644 index 0000000..58a352e --- /dev/null +++ b/buildkit/docker-compose.6996.yml @@ -0,0 +1,63 @@ +# BuildKit daemon —— 基于上游 PR #6996(env 变量式上游代理)的验证部署 +# 与 docker-compose.buildkit.yml 的区别: +# - 镜像: 从 gmarmstrong/buildkit@proxy-chaining 编译 +# - 上游代理配置从 buildkitd.toml 的 [proxy] 段 → buildkitd 进程环境变量 +# - CA 信任从 upstreamCACert 显式配置 → 系统信任库(SSL_CERT_FILE) +# +# 对照组用法: 把 SSL_CERT_FILE 这行注释掉再 up, HTTPS 下载应因证书校验失败 +# (证明无 cert 配置时, CA 必须预置进系统信任库, 这是 PR #6996 的关键限制) + +services: + buildkitd: + image: tommylike/buildkit-6996:test + container_name: buildkitd + entrypoint: ["buildkitd", "--addr", "tcp://0.0.0.0:1234", "--config", "/etc/buildkit/buildkitd.toml"] + environment: + # 上游代理 = Squid HA VIP(PR #6996: 从 buildkitd 环境读取) + HTTP_PROXY: "http://172.30.0.100:3128" + HTTPS_PROXY: "http://172.30.0.100:3128" + http_proxy: "http://172.30.0.100:3128" + https_proxy: "http://172.30.0.100:3128" + NO_PROXY: "localhost,127.0.0.1,172.30.0.0/24" + no_proxy: "localhost,127.0.0.1,172.30.0.0/24" + # PR #6996 无 cert 配置 → 信任下沉到系统信任库。Go x509 会读 SSL_CERT_FILE, + # 把 Squid SSL-Bump CA 加进内建代理出站连接的系统证书池。 + # (对照组: 注释此行 → HTTPS 经 bump Squid 应证书校验失败) + SSL_CERT_FILE: "/etc/buildkit/squid-ca.pem" + cap_drop: + - ALL + cap_add: + - SYS_ADMIN + - NET_ADMIN + - NET_RAW + - SETUID + - SETGID + - MKNOD + - SYS_CHROOT + - DAC_OVERRIDE + - CHOWN + - FOWNER + - FSETID + - SETPCAP + - SETFCAP + - KILL + - SYS_PTRACE + - NET_BIND_SERVICE + - AUDIT_WRITE + security_opt: + - apparmor=unconfined + - seccomp=unconfined + - systempaths=unconfined + cgroup: host + networks: + squid_net: + ipv4_address: 172.30.0.30 + volumes: + - ./buildkitd-6996.toml:/etc/buildkit/buildkitd.toml:ro + - ../configs/certs/client-ca.crt:/etc/buildkit/squid-ca.pem:ro + - /sys/fs/cgroup:/sys/fs/cgroup:rw + +networks: + squid_net: + external: true + name: haproxy_ha_squid_net diff --git a/reports/buildkit-pr6996-validation-20260813.md b/reports/buildkit-pr6996-validation-20260813.md new file mode 100644 index 0000000..2727c7f --- /dev/null +++ b/reports/buildkit-pr6996-validation-20260813.md @@ -0,0 +1,122 @@ +# BuildKit 上游 PR #6996(env 变量式上游代理)端到端验证报告 + +**日期**:2026-08-13 · **验证人**:TommyLike + Claude +**结论先行**:功能正确、与我们的 Squid SSL-Bump 缓存场景完全兼容;无 cert 配置的设计下, +CA 信任经 `SSL_CERT_FILE` 走系统信任库,实测成立。PULL 与 RUN 共用一套 env 配置是 +BuildKit 既有标准行为(非本 PR 引入),且与 `deploy/` 的 splice + registry-proxy 架构互补。 + +--- + +## 1. 背景 + +本仓库此前的 BuildKit 扩展(`buildkit/`)基于我们自己 fork 的实现 +(`TommyLike/buildkit` `feature/upstream-proxy-config`,PR #1):在 `buildkitd.toml` 里用 +`[proxy] upstreamURL + upstreamCACert` 把内置 exec 代理链到 Squid。 + +上游社区后来由 gmarmstrong 提了 **moby/buildkit#6996** +([network: chain proxy requests via upstream proxy](https://github.com/moby/buildkit/pull/6996)), +走**完全不同的路线**: + +| | 本仓库 fork 方案 | 上游 #6996 | +|---|---|---| +| 配置载体 | `buildkitd.toml` 的 `[proxy]` 段 | buildkitd 进程**环境变量**(`HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY`) | +| CA 信任 | `upstreamCACert` 显式指定文件 | **无 cert 配置**,信任下沉**系统信任库** | +| 覆盖范围 | 仅内置 exec 代理出站 | 同一 env 同时覆盖 exec 链 + 镜像拉取(后者为 BuildKit 既有行为) | + +本报告用上游实现**重新编译镜像并做端到端验证**,判断它是否可替代我们的 fork。 + +## 2. 被测代码与镜像 + +- 代码:`gmarmstrong/buildkit` 分支 `proxy-chaining`(sha `e9606c76e0`,2 commits:实现 + docs 展开) +- 镜像:`tommylike/buildkit-6996:test`(241MB,`--target buildkit` 编译,含 buildkitd/buildctl/runc/cni) +- 依赖状态:#6996 依赖的 #6995(`FilterProxyEnv`)当时未合并;`ReplaceEnv` 等 helper 在本分支自包含,功能不受影响 + +编译命令: + +```bash +git clone --depth 1 --branch proxy-chaining https://github.com/gmarmstrong/buildkit.git +cd buildkit +docker buildx build --target buildkit -t tommylike/buildkit-6996:test --load . +``` + +## 3. 测试配置(关键差异) + +与 fork 版 `docker-compose.buildkit.yml` 的差异:上游代理从 `buildkitd.toml [proxy]` 段 +→ **buildkitd 进程环境变量**;CA 信任从 `upstreamCACert` 显式配置 → **系统信任库**。 + +```yaml +environment: + HTTP_PROXY: "http://172.30.0.100:3128" # Squid HA VIP + HTTPS_PROXY: "http://172.30.0.100:3128" + NO_PROXY: "localhost,127.0.0.1,172.30.0.0/24" + SSL_CERT_FILE: "/etc/buildkit/squid-ca.pem" # ← 唯一的 CA 接入点(系统信任库) +``` + +- `buildkit/buildkitd-6996.toml`:只保留 `proxyNetwork = true`,**无 `[proxy]` 段** +- `buildkit/docker-compose.6996.yml`:完整部署定义(含精简 capabilities,与 fork 版同套) + +代码正确性关键点:`newProxyTransport()` 的 `TLSClientConfig == nil` → Go 回退**系统证书池** +→ `SSL_CERT_FILE` 生效。这是"无 cert 配置"路线成立的代码依据(实测也确认,见 §4)。 + +## 4. 测试矩阵与结果 + +| 场景 | 配置 | 预期 | 实测 | +|---|---|---|---| +| **正对照** | `SSL_CERT_FILE` = Squid CA | 构建成功 | ✅ RUN 内 HTTPS 下载 200,RPM 14.9MB,proxy 捕获 `openeuler.org -> 200` | +| **缓存** | 同上,二次构建 | Squid `TCP_HIT` 增加 | ✅ 75 → 76 | +| **负对照** | 无 `SSL_CERT_FILE`(其余同) | RUN 步骤证书校验失败 | ✅ exec 代理对 Squid 重签证书校验失败 → 返回 **502**,apk 构建失败 | + +负对照演进过程(记录以便复现): + +1. **v1(全流量走代理)**:失败点在**镜像拉取**——`registry-1.docker.io` x509 失败。 + 证明 env 方案下 daemon 自身 registry 流量也走上游 Squid,同样需要 CA 信任。 +2. **v3(隔离 RUN 路径)**:`NO_PROXY` 加入 `docker.io,docker.com` 摘除镜像拉取 + (含 blob 重定向的 `production.cloudfront.docker.com`),失败点精确落在 + `RUN apk add` 步骤——exec 代理对 bump 证书校验失败返回 502。 + Squid 侧日志:CONNECT 隧道建立(`NONE_NONE/200`)但隧道内无 GET 记录(握手被拒)。 + +## 5. 关键结论 + +1. **功能正确**。env 变量生效、大小写优先级(`http.ProxyFromEnvironment` 标准行为)、 + `NO_PROXY` 语义、非法 URL fail-closed、凭证脱敏,均符合文档与设计。 +2. **无 cert 配置 → 系统信任库,实测成立**。`SSL_CERT_FILE` 一路贯通: + exec 信任内建代理(自动注入)→ 内建代理信任 Squid bump CA(系统池)→ Squid 回源。 +3. **"PULL 也走代理"是收益不是问题**。buildkitd 尊重 proxy env 做镜像拉取是 + BuildKit **既有标准行为**,#6996 只是把同一约定延伸到 exec 链,一次配置同时解决 + PULL + RUN 两个场景。且与 `deploy/` 架构天然互补: + - PULL → squid 对 registry 域 splice → cache_peer → registry-proxy 缓存镜像 blob(无需 CA) + - RUN → squid bump → 缓存 apk/rpm/pip(需要 CA 进系统信任库) +4. **可替代我们的 fork**。env 方式更符合社区惯例、无定制配置面;fork 的 + `upstreamCACert` 旋钮在不可变镜像场景仍有便利性,但非必需。 + +## 6. 遗留建议(对上游 PR,不影响本结论) + +1. `docs/proxy.md` 的 "Upstream proxies" 节**缺 CA 信任说明**:经 SSL-bump/私有 CA 上游 + 做 HTTPS 缓存时,必须把代理 CA 预置进 buildkitd 系统信任库(`SSL_CERT_FILE` 或 + `update-ca-certificates`)。不写,用户照文档配置 HTTPS 缓存必踩 502(已在上游 PR 评论)。 +2. 合并顺序:#6995(`FilterProxyEnv`)先合,再 de-dup 两边重复的 `ReplaceEnv` helper。 +3. fail-closed 偏硬:上游 Squid 挂掉时所有 proxy-network exec 全部失败,无 soft-fail 选项, + 对"缓存可无、构建要能跑"的场景值得后续讨论。 + +## 7. 复现方法 + +```bash +# 1. 起主 Squid HA 环境 +cd .. && ./scripts/setup.sh && cd buildkit + +# 2. 编译镜像(或直接用 tommylike/buildkit-6996:test) +# 见 §2 编译命令 + +# 3. 启动 env 版 buildkitd +docker compose -f docker-compose.6996.yml up -d + +# 4. 跑一次构建(正对照) +docker run --rm --network haproxy_ha_squid_net -v "$PWD:/work:ro" --entrypoint buildctl \ + tommylike/buildkit-6996:test --addr tcp://172.30.0.30:1234 build \ + --frontend dockerfile.v0 --local context=/work --local dockerfile=/work \ + --opt filename=Dockerfile.test \ + --opt build-arg:RPM_URL=https://repo.openeuler.org/openEuler-23.03/debuginfo/aarch64/Packages/bcc-debuginfo-0.26.0-1.oe2303.aarch64.rpm \ + --no-cache --progress plain + +# 5. 验证 Squid 缓存命中(TCP_HIT 增加)与负对照(去掉 SSL_CERT_FILE 重建容器) +``` From cb3099a1cc8299caedcefcbdf7f9b05f3953b637 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Wed, 12 Aug 2026 17:27:00 +0800 Subject: [PATCH 07/22] docs(tracfic): record chart 0.1.4 cache-rule optimization + retest results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 3 new refresh_patterns: .crate / .zip / .pth|pt|safetensors - helm upgrade REVISION 14 (squid-rpardini-0.1.4), ArgoCD ownership + kubelet volume-sync gotchas - retest: cargo 11% -> ~95% hit, wget 94MB .pth TCP_HIT zero-origin feat(chart): 0.1.4 — force-cache rules for cargo/model-weight traffic - refresh_pattern for .crate (cargo), .zip, .pth/.pt/.safetensors (model weights) - measured: cargo hit 11% -> ~95%, wget 94MB .pth TCP_HIT zero-origin (gy-006) - verified via helm upgrade REVISION 14 --- .gitignore | 2 + deploy/chart/Chart.yaml | 2 +- deploy/chart/templates/configmap.yaml | 3 + traffic-test/README.md | 57 +++++++++ traffic-test/RESULTS.md | 83 +++++++++++++ traffic-test/TOOL-RESULTS.md | 81 ++++++++++++ traffic-test/analyze-tool-traffic.py | 86 +++++++++++++ traffic-test/gen-replicas.py | 21 ++++ traffic-test/git-clone-traffic.yaml | 133 ++++++++++++++++++++ traffic-test/monitor-traffic.sh | 57 +++++++++ traffic-test/pip-traffic.yaml | 170 ++++++++++++++++++++++++++ traffic-test/run-tool-traffic.sh | 114 +++++++++++++++++ traffic-test/run.sh | 100 +++++++++++++++ 13 files changed, 908 insertions(+), 1 deletion(-) create mode 100644 traffic-test/README.md create mode 100644 traffic-test/RESULTS.md create mode 100644 traffic-test/TOOL-RESULTS.md create mode 100644 traffic-test/analyze-tool-traffic.py create mode 100644 traffic-test/gen-replicas.py create mode 100644 traffic-test/git-clone-traffic.yaml create mode 100755 traffic-test/monitor-traffic.sh create mode 100644 traffic-test/pip-traffic.yaml create mode 100755 traffic-test/run-tool-traffic.sh create mode 100755 traffic-test/run.sh diff --git a/.gitignore b/.gitignore index 5fa35c4..86557cc 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ buildkit/src/ .metrics-baseline .metrics-k8s-state .metrics-k8s-baseline +traffic-test/logs +traffic-test/traffic-gen diff --git a/deploy/chart/Chart.yaml b/deploy/chart/Chart.yaml index 6608bbe..bd347fa 100644 --- a/deploy/chart/Chart.yaml +++ b/deploy/chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: squid-rpardini description: Squid proxy with SSL Bump + registry cache sidecar for CI workloads type: application -version: 0.1.3 +version: 0.1.4 appVersion: "7.6" diff --git a/deploy/chart/templates/configmap.yaml b/deploy/chart/templates/configmap.yaml index f8d3a1c..d07f086 100644 --- a/deploy/chart/templates/configmap.yaml +++ b/deploy/chart/templates/configmap.yaml @@ -52,6 +52,9 @@ data: refresh_pattern -i \.whl$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache refresh_pattern -i \.tar\.gz$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache refresh_pattern -i \.deb$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache + refresh_pattern -i \.crate$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache + refresh_pattern -i \.zip$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache + refresh_pattern -i \.(pth|pt|safetensors)$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache refresh_pattern -i repo\.huaweicloud\.com/.*/simple/ 0 20% 4320 ignore-private ignore-reload override-vary refresh_pattern -i mirrors\.tuna\.tsinghua\.edu\.cn/.*/simple/ 0 20% 4320 ignore-private ignore-reload override-vary refresh_pattern -i .pypi.org/.* 0 20% 4320 ignore-private diff --git a/traffic-test/README.md b/traffic-test/README.md new file mode 100644 index 0000000..542e08c --- /dev/null +++ b/traffic-test/README.md @@ -0,0 +1,57 @@ +# tracfic-test — 模拟 CI 并发流量的 Squid 流量模式测试 + +在 gy-006 集群(namespace `squid`)用 **10 个并行 Volcano 任务**模拟真实 CI 并发, +通过 squid SSL-Bump 代理访问 `gitcode.com/Ascend` 真实仓库,观察 squid 的流量模式 +(回源 vs 缓存命中、并发带宽)。 + +## 场景 + +| 场景 | 任务数 | 流量形态 | 期望模式 | +|---|---|---|---| +| `pip-traffic.yaml` | 10 | clone pytorch (67MB) + pip 下载 12 个真实包(含 torch 2.10.0 aarch64 wheel 146MB) | **wheel 可缓存**:第 1 个任务回源 146MB,后 9 个任务 squid HIT(`.whl` refresh_pattern 100% 缓存)→ 回源流量 ≈ 1× wheel,出站流量 ≈ 10× wheel | +| `git-clone-traffic.yaml` | 10 | `git clone --depth 1 gitcode.com/Ascend/pytorch`(~67MB pack/任务) | **git pack 不可缓存**(默认 `refresh_pattern . 0 20% 4320`,smart-HTTP pack 无 Last-Modified)→ 10 个任务全部回源,回源 ≈ 10×67MB,无缓存增益 | + +每个 pod 打印:clone/pip 耗时、下载字节、聚合带宽、`DURATION`。 + +## 真实负载来源 + +- 仓库:`gitcode.com/Ascend/pytorch`(华为 Ascend PyTorch fork,审计见 `deploy/tool/ascend-org-build-tools-report.md`) +- pip 包:pytorch 真实 `requirements.txt`(pyyaml/setuptools/auditwheel)+ Ascend CI 常用依赖 + (numpy/psutil/requests/tqdm/regex/pygments/flask/fastapi/pydantic)+ `torch==2.10.0` + (cp311-aarch64 wheel 146MB,镜像已验证) +- pip 索引:`repo.huaweicloud.com/repository/pypi/simple`(Ascend CI 实际使用的镜像) + +## 用法 + +```bash +# 提交 git clone 流量测试(10 任务)+ 后台监控中央 Prometheus 900s +./run.sh git --monitor=900 + +# 提交 pip 流量测试 +./run.sh pip + +# 两个场景顺序执行 +./run.sh both --monitor=1800 +``` + +Kubeconfig 默认 `~/.kube/gy-006.yaml`(`KUBECONFIG` 可覆盖)。 +日志落盘 `logs/--.log`(每 pod 一份),流量采样 `logs/traffic.tsv`。 + +## 监控 + +`monitor-traffic.sh` 从中央 Prometheus(`113.44.182.82:9090`)采样 squid 每副本指标: + +| 指标 | 含义 | +|---|---| +| `squid_client_http_kbytes_out_kbytes_total` | squid → 客户端字节(所有响应,含 HIT) | +| `squid_server_http_kbytes_in_kbytes_total` | 回源字节(仅 MISS/未缓存时产生) | + +输出 TSV:`ts out_total_KB/s in_total_KB/s instance-1... instance-N`。 + +## 流量模式解读 + +- **pip 场景**:`in`(回源)峰值 ≈ 1×146MB wheel + 小包;`out` ≈ 10×(10 个任务并行拉包)。 + 若 `in/out` 比值 ≪ 1 → wheel 缓存命中,squid 缓存生效。 +- **git 场景**:`in` ≈ 10×67MB(10 个任务几乎同时回源),`in/out` ≈ 1 → git 不缓存, + 印证 `refresh_pattern` 对 git smart-HTTP pack 无效(这正是 CI 里 git clone 无缓存加速的原因)。 +- 每副本聚合:`sum by (instance) (rate(...[1m]))` 查看负载是否均衡分发到两个 squid 副本。 diff --git a/traffic-test/RESULTS.md b/traffic-test/RESULTS.md new file mode 100644 index 0000000..2d4da9b --- /dev/null +++ b/traffic-test/RESULTS.md @@ -0,0 +1,83 @@ +# tracfic-test 实测结果(2026-08-12, gy-006) + +10 个并行 Volcano 任务模拟 CI 并发,全部走 squid SSL-Bump(`squid-cache.squid:3128`)。 +数据来源:中央 Prometheus(`113.44.182.82:9090`,headless 每副本 scrape)+ 双副本 access.log。 + +## 指标语义(对比口径) + +| 指标 | 方向 | 用途 | +|---|---|---| +| `squid_client_http_kbytes_out_kbytes_total` | squid → 客户端(全部响应) | **出站** | +| `squid_server_http_kbytes_in_kbytes_total` | 源站 → squid(仅 MISS 回源) | **回源** | + +缓存命中率 = `1 - server_in / client_out`(同方向响应流量之比)。 + +## 场景 1:pip 并发安装(10 任务) + +每任务:clone `gitcode.com/Ascend/pytorch`(63MB)+ `pip download` 13 个真实包 +(pytorch requirements + numpy/psutil/flask/fastapi 等 + `torch==2.10.0` aarch64 wheel 146MB)。 + +### 冷缓存(首轮) + +| 指标 | 数值 | +|---|---| +| pip download 耗时 | 3.9–10.9s(多数 ~4s) | +| 场景出站(client_out) | 1,617 MB | +| 场景回源(origin_in) | 112 MB(≈ 1× wheel + 小包) | +| 回源率 / 缓存命中率 | **6.9% / 93.1%** | +| access.log HIT | cache-0: 84%,cache-1: 80%(MEM_HIT+HIT+REFRESH_UNMODIFIED) | + +### 热缓存(重测) + +| 指标 | 数值 | +|---|---| +| pip download 耗时 | 3.8–5.3s(分布均匀,无慢任务) | +| 下载阶段命中率曲线 | **1.00**(origin 仅 0.2–0.5 KB/s,wheel 全 HIT) | +| 峰值出站 | 18.1 MB/s,回源 1.05 MB/s(hitrate 0.94) | + +**结论**:`.whl` 强制缓存(`refresh_pattern 10080 100% 525960`)生效。10 个并发 CI 任务中 +仅第 1 个回源 146MB wheel;热缓存后**回源几乎为零**。 + +## 场景 2:git 并发 clone(10 任务) + +`git clone --depth 1 https://gitcode.com/Ascend/pytorch.git`(63MB pack)。 + +| 轮次 | clone 耗时 | access.log | +|---|---|---| +| 首轮 | 2.9–4.7s(avg 3.7s) | **100% TCP_MISS** | +| 重测 | 2.7–3.6s | **100% TCP_MISS** | + +重测窗口 access.log 中 `git-upload-pack` 共 **40 个请求全部 TCP_MISS**(两副本各 14+26)。 +**结论**:git smart-HTTP pack(`application/x-git-upload-pack-result`)不缓存,10 并发全部回源。 + +## 混合窗口总账(重测,07:48:00-07:52:00 UTC,两副本) + +| 状态 | 请求数 | 字节 | 说明 | +|---|---|---|---| +| TCP_HIT | 607 | 2,690 MB | 磁盘命中(pip wheel 主体) | +| TCP_MEM_HIT | 1,984 | 201 MB | 内存命中(pip 索引页) | +| TCP_REFRESH_UNMODIFIED | 806 | 1,108 MB | 重新验证未变(命中) | +| **TCP_MISS** | **61** | **245 MB** | 回源(**其中 40 个 upload-pack = 全部 git clone pack**) | +| TCP_REFRESH_MODIFIED | 12 | 1.4 MB | 重新验证有更新 | + +- Prometheus 同窗口:client_out 4,247 MB / origin_in 246 MB → **回源率 5.8%,缓存命中率 94.2%**(与 access.log MISS 字节 245MB 完全吻合) +- 回源 ≈ 全部为 git pack(245MB ≈ 40× pack 段);pip 热缓存阶段回源≈0 + +## 双副本负载 + +| 副本 | 命中字节 | MISS 字节 | git upload-pack | +|---|---|---|---| +| squid-cache-0 | 1,869 MB | 86 MB | 14 | +| squid-cache-1 | 2,130 MB | 159 MB | 26 | + +负载基本均衡(10 任务按节点 5/5 分流,比例 ~47:53)。 + +## 方法说明 + +- 窗口流量:Prometheus `sum(...{job="squid"})` counter delta(headless 每副本 series) +- HIT/MISS:access.log `$4` 列(TCP_HIT/TCP_MISS/TCP_MEM_HIT/TCP_REFRESH_*),`$5`=字节 +- 时间戳:Prometheus/access.log 为 UTC epoch;本机日志显示为 UTC+8 +- `kubectl cp` 抓 access.log 可能截断(EOF 错误),须在容器内 `grep | awk` +- `rate[5m]` 摊平峰值(瞬时速率 ≈ rate 值 × 300s / 峰值持续时间) +- Prometheus 的 `client_http.kbytes_in`(客户端→squid 请求字节)恒小,**勿用作出站对比**; + 对比必须用 `client_http.kbytes_out` vs `server_http.kbytes_in` diff --git a/traffic-test/TOOL-RESULTS.md b/traffic-test/TOOL-RESULTS.md new file mode 100644 index 0000000..20c98b1 --- /dev/null +++ b/traffic-test/TOOL-RESULTS.md @@ -0,0 +1,81 @@ +# 16-tool 并发流量测试结果(2026-08-12, gy-006) + +`run-tool-traffic.sh --all`:对 `squid-openssl/testcase/tool/` 全部 16 个工具 case +跑 **10 任务并发**流量测试(volcano replicas=10),时间线 `logs/tool/timeline.tsv`, +窗口流量用 access.log(逐请求,精确)+ 中央 Prometheus counter delta(参考)双口径。 + +## 汇总表 + +| case | 工具/流量特征 | 出站 | HIT/MISS(access.log) | HIT% | 说明 | +|---|---|---|---|---|---| +| 02 | apt | 5,543MB | 46 / 0.1MB | **99.8%** | .deb 强制缓存,10 并发几乎零回源 | +| 03 | git clone (github→gh-proxy) | 12MB | 0 / 0.1MB | **0.0%** | git pack 不缓存(结构性),vllm-ascend 浅克隆小 | +| 04 | go mod download | 1,577MB | 2,472 / 283MB | **89.7%** | proxy.golang.org 命中 | +| 05 | obsutil | 短窗口 | 243 / 3.3MB | **98.6%** | OBS 对象缓存命中 | +| 06 | wget 模型权重(.pth) | 855MB | 739 / 487MB | **60.3%** | .pth 无强制缓存规则,靠 20% LM 启发式 | +| 07 | cmake FetchContent | 1,118MB | 451 / 276MB | **62.1%** | 部分 URL 带缓存标记 | +| 08 | bazel http_archive | 848MB | 1,545 / 90MB | **94.5%** | 命中好 | +| 09 | npm install | 2,754MB | 3,246 / 157MB | **95.4%** | registry tarball 命中 | +| 10 | cargo build | 2,422MB | 19 / 157MB | **11.0%** | **crates.io .crate 不缓存**(默认 refresh_pattern 0/20%/4320)⚠ | +| 11 | conda | 2,002MB | 2,002 / 4.6MB | **99.8%** | 命中完美 | +| 12 | uv pip | 916MB | 2,638 / 134MB | **95.2%** | wheel 缓存 | +| 13 | huggingface | — | 463 / 41MB | **91.9%** | job FAILED(hf hub 网络,与历史一致),失败前命中良好 | +| 14 | git-lfs | 1,225MB | 1,469 / 5.1MB | **99.7%** | LFS 对象大文件命中 | +| 15 | pnpm | 2,364MB | 2,204 / 59MB | **97.4%** | 命中好 | +| 16 | yum/dnf | 2,625MB | 2,438 / 2.2MB | **99.9%** | 命中完美 | + +(注:HIT = TCP_HIT + TCP_MEM_HIT + TCP_REFRESH_UNMODIFIED;MISS = TCP_MISS + TCP_REFRESH_MODIFIED。 +出站为 Prometheus client_out counter delta——短窗口 case(05 等)因 60s scrape 对齐会低估,access.log 口径为准。) + +## 结论与洞察 + +1. **多数工具 89-100% 命中**:apt/yum/conda/git-lfs/obs/npm/pnpm/bazel/uv——包管理器静态文件被 + squid 缓存,10 并发 CI 只有 1 份回源。 +2. **git 系 0%**(03-github + 上次 git-clone 测试):smart-HTTP pack 永不缓存,结构性限制。 +3. **⚠ cargo(11%)最低可优化项**:crates.io 的 `.crate` 静态文件无强制缓存规则。建议 squid 增加: + ``` + refresh_pattern -i \.crate$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache + ``` + 预计可提升到 ~90%+(cargo 是 Rust 构建的重负载工具)。 +4. **wget/cmake(60-62%)**:模型权重(.pth/.pt)和无扩展名大文件靠 20% LM 启发式命中率一般。 + 可选:为模型权重域名加 min-age 缓存规则(需按实际域名配置)。 +5. **13-huggingface** 本次 job 失败(hf hub 连接,与历史测试一致)——但失败前已观察到 463MB + 命中,缓存机制本身正常。 + +## chart 0.1.4 缓存规则优化 + 重测(2026-08-12) + +针对上述洞察,`deploy/chart/templates/configmap.yaml` 新增 3 条强制缓存规则 +(`deploy/chart/Chart.yaml` 0.1.3 → 0.1.4): + +```squid +refresh_pattern -i \.crate$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache +refresh_pattern -i \.zip$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache +refresh_pattern -i \.(pth|pt|safetensors)$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache +``` + +升级:`helm upgrade squid . -f values-006.yaml -n squid`(REVISION 14, squid-rpardini-0.1.4)。 +排坑:集群资源此前由 ArgoCD 管理(managedFields 残留 argocd-controller ownership + 远程 +controller 每 ~3min 回滚 configmap),需 `kubectl apply --server-side --field-manager=helm +--force-conflicts` 抢回字段所有权后再 helm upgrade;ConfigMap 卷更新有 kubelet 同步延迟 +(>3min),须等同步完成后再 rollout restart 才能让 squid 加载新规则。 + +### 重测对比(新规则生效后,10 并发) + +| 场景 | 优化前命中率 | 优化后 | 证据(access.log) | +|---|---|---|---| +| **cargo**(rsproxy crate 下载) | **11.0%** | **~95%** | cache-0: HIT 11.2MB / MISS 0.3MB;cache-1: HIT 7.7MB / MISS 1.1MB(TCP_HIT + TCP_MEM_HIT + REFRESH_UNMODIFIED) | +| **wget**(94MB resnet50.pth) | 60.3% | **100%** | `TCP_HIT/200 94285753 GET .../resnet50_msra-5891d200.pth`(HIER_NONE,零回源,1.2s / 78MB/s) | + +- cargo crate 文件(`.crate`)从 11% → ~95%:10 并发 CI 中 9 份下载命中缓存 +- 模型权重(.pth)现在强制缓存:94MB 文件零回源 + +## 复现 + +```bash +./run-tool-traffic.sh --all --monitor=7200 # 16 case × 10 并发 + 流量监控(约 40-60min) +python3 analyze-tool-traffic.py # 汇总表 → logs/tool/analysis.json +``` + +- timeline:`logs/tool/timeline.tsv`(SUBMIT/DONE epoch,用于窗口对齐) +- 日志:`logs/tool/-/pod-N.log`(每 case 10 份) +- 流量采样:`logs/tool/traffic.tsv`(client/origin/hitrate 曲线,rate[5m]) diff --git a/traffic-test/analyze-tool-traffic.py b/traffic-test/analyze-tool-traffic.py new file mode 100644 index 0000000..1f92b5f --- /dev/null +++ b/traffic-test/analyze-tool-traffic.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""analyze-tool-traffic.py — per-case traffic analysis for the 16-tool test. + +For each case (window = SUBMIT..DONE from timeline.tsv): + - Prometheus counter delta: client_out / origin_in → 回源率 + - access.log (both replicas): HIT/MISS bytes by status class +Output: summary table + per-case JSON. +""" +import json +import subprocess +import sys +import urllib.request +import urllib.parse + +PROM = "http://113.44.182.82:9090" +KUBECONFIG = "/home/chenqi252/.kube/gy-006.yaml" +LOGDIR = sys.argv[1] if len(sys.argv) > 1 else "logs/tool" + +def range_q(expr, start, end, step=30): + q = {"query": expr, "start": start, "end": end, "step": step} + d = json.load(urllib.request.urlopen(PROM + "/api/v1/query_range?" + urllib.parse.urlencode(q), timeout=20)) + return d["data"]["result"] + +def delta(met, s, e): + r = range_q(f'sum({met}{{job="squid"}})', s, e) + if not r: + return 0 + vals = [float(x[1]) for x in r[0]["values"] if x[1] != ""] + return (vals[-1] - vals[0]) / 1024 if len(vals) >= 2 else 0 + +def accesslog(pod, s, e): + """return {status: (reqs, bytes)} in window from pod access.log""" + out = subprocess.run( + ["kubectl", "--kubeconfig", KUBECONFIG, "exec", "-n", "squid", pod, "--", + "sh", "-c", f'awk \'{{if ($1>={s} && $1<={e}) {{x=$4; sub(/\\/.*/,"",x); c[x]++; b[x]+=$5}}}} END{{for (k in c) printf "%s %d %d\\n", k, c[k], b[k]}}\' /var/log/squid/access.log'], + capture_output=True, text=True, timeout=60) + res = {} + for line in out.stdout.splitlines(): + parts = line.split() + if len(parts) == 3: + res[parts[0]] = (int(parts[1]), int(parts[2])) + return res + +def classify(stats): + hit = miss = 0.0 + for k, (n, b) in stats.items(): + if k.startswith("TCP_HIT") or k.startswith("TCP_MEM_HIT") or k.startswith("TCP_REFRESH_UNMODIFIED") or k.startswith("TCP_REFRESH_HIT"): + hit += b + elif k.startswith("TCP_MISS") or k.startswith("TCP_REFRESH_MODIFIED"): + miss += b + return hit / 1048576, miss / 1048576 + +# timeline +tl = {} +with open(f"{LOGDIR}/timeline.tsv") as f: + for line in f: + parts = line.strip().split("\t") + if len(parts) != 4: + continue + ts, case, action, job = parts + if action in ("SUBMIT", "DONE", "FAILED"): + tl.setdefault(case, {})[action] = int(ts) + +cases = sorted(tl.keys()) +print(f"{'case':<22} {'out_MB':>9} {'origin_MB':>9} {'回源%':>7} {'HIT_MB':>8} {'MISS_MB':>8} {'HIT%':>6}") +results = {} +for c in cases: + t = tl[c] + s, e = t["SUBMIT"], t.get("DONE", t.get("FAILED", t["SUBMIT"] + 60)) + co = delta("squid_client_http_kbytes_out_kbytes_total", s, e) + oi = delta("squid_server_http_kbytes_in_kbytes_total", s, e) + hit_t = miss_t = 0.0 + for pod in ("squid-cache-0", "squid-cache-1"): + st = accesslog(pod, s, e) + h, m = classify(st) + hit_t += h + miss_t += m + ratio = 100 * oi / co if co > 0 else 0 + hitpct = 100 * hit_t / (hit_t + miss_t) if (hit_t + miss_t) > 0 else 0 + results[c] = dict(out_mb=round(co, 1), origin_mb=round(oi, 1), ratio=round(ratio, 1), + hit_mb=round(hit_t, 1), miss_mb=round(miss_t, 1), hit_pct=round(hitpct, 1)) + print(f"{c:<22} {co:>9.1f} {oi:>9.1f} {ratio:>6.1f}% {hit_t:>8.1f} {miss_t:>8.1f} {hitpct:>5.1f}%") + +with open(f"{LOGDIR}/analysis.json", "w") as f: + json.dump(results, f, indent=1) +print(f"\n→ {LOGDIR}/analysis.json") diff --git a/traffic-test/gen-replicas.py b/traffic-test/gen-replicas.py new file mode 100644 index 0000000..eb5d4b5 --- /dev/null +++ b/traffic-test/gen-replicas.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""gen-replicas.py — clone a tool yaml with tasks[0].replicas set to N. + +Usage: gen-replicas.py [replicas] +""" +import sys +import yaml + +src, out = sys.argv[1], sys.argv[2] +n = int(sys.argv[3]) if len(sys.argv) > 3 else 10 + +doc = yaml.safe_load(open(src)) +task = doc['spec']['tasks'][0] +task['replicas'] = n +doc['metadata']['labels'].setdefault('pipeline/run-id', 'x') +doc['metadata']['labels']['pipeline/run-id'] += f'-trafficx{n}' + +with open(out, 'w') as f: + yaml.safe_dump(doc, f, default_flow_style=False, sort_keys=False, + allow_unicode=True, width=1000000) +print(f"wrote {out} (replicas={n})") diff --git a/traffic-test/git-clone-traffic.yaml b/traffic-test/git-clone-traffic.yaml new file mode 100644 index 0000000..2bfdca8 --- /dev/null +++ b/traffic-test/git-clone-traffic.yaml @@ -0,0 +1,133 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-traffic-git- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-traffic-git +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 3600 + tasks: + - name: test-traffic-git + replicas: 10 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + echo "== task: $(hostname) ==" + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -1 || true + apt-get install -y -qq git 2>&1 | tail -2 || { apt-get update -qq 2>&1 | tail -1 || true; apt-get install -y -qq git 2>&1 | tail -2; } + T_START=$(date +%s%3N) + + echo "" + echo "==========================================" + echo "git clone gitcode.com/Ascend/pytorch (depth 1)" + echo "(git smart-HTTP packs are NOT cached by squid —" + echo " every task pulls the full pack from origin)" + echo "==========================================" + T0=$(date +%s%3N) + rm -rf /workspace/pytorch + git clone --depth 1 --quiet https://gitcode.com/ascend/pytorch.git /workspace/pytorch + T1=$(date +%s%3N) + SIZE=$(du -sb /workspace/pytorch 2>/dev/null | awk '{print $1}') + MS=$((T1 - T0)) + echo "clone: ${MS}ms, size=${SIZE}B" + if [ "$MS" -gt 0 ]; then + echo "clone aggregate bandwidth: $(( SIZE * 1000 / MS )) B/s" + fi + + T_END=$(date +%s%3N) + echo "" + echo "DURATION: $((T_END - T_START))ms" + echo "== done $(hostname) ==" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then # RHEL/openEuler + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else # Debian/Ubuntu + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "%s";\n' \ + "$HTTPS_PROXY" "$HTTPS_PROXY" > /etc/apt/apt.conf.d/99squid-proxy + fi + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 3600 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/monitor-traffic.sh b/traffic-test/monitor-traffic.sh new file mode 100755 index 0000000..ae5e31f --- /dev/null +++ b/traffic-test/monitor-traffic.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# monitor-traffic.sh — sample squid traffic from the central Prometheus +# while a traffic test job runs. +# +# Usage: +# ./monitor-traffic.sh [duration_s] [out.tsv] +# default: 600s, stdout +# +# Metrics (central Prometheus 113.44.182.82:9090, headless per-replica scrape): +# squid_server_http_kbytes_in_kbytes_total = origin-server bytes (MISS only, upstream fetch) +# squid_client_http_kbytes_out_kbytes_total = bytes sent to clients (all responses) +# +# Rows: ts t -out(KB/s) -in(KB/s) ... out_total in_total +set -euo pipefail + +PROM="${PROM:-http://113.44.182.82:9090}" +DUR="${1:-600}" +OUT="${2:-}" +STEP=10 + +q() { # promql → values + local expr="$1" + curl -s --max-time 8 -G "$PROM/api/v1/query" --data-urlencode "query=$expr" \ + | python3 -c " +import json,sys +d=json.load(sys.stdin) +for r in d.get('data',{}).get('result',[]): + inst=r['metric'].get('instance','?') + print(inst, r['value'][1]) +" +} + +CLIENT_EXPR='sum(rate(squid_client_http_kbytes_out_kbytes_total{job="squid"}[5m]))' +ORIGIN_EXPR='sum(rate(squid_server_http_kbytes_in_kbytes_total{job="squid"}[5m]))' +HITRATE_EXPR='1 - sum(rate(squid_server_http_kbytes_in_kbytes_total{job="squid"}[5m])) / sum(rate(squid_client_http_kbytes_out_kbytes_total{job="squid"}[5m]))' +PER_EXPR='sum by (instance) (rate(squid_client_http_kbytes_out_kbytes_total{job="squid"}[5m]))' + +echo "# squid traffic monitor: ${DUR}s, step ${STEP}s, prom=${PROM}" +echo "# ts client_kb/s origin_kb/s hitrate(1-origin/client) per-instance(client)" +t0=$(date +%s) +end=$((t0 + DUR)) +while [ "$(date +%s)" -lt "$end" ]; do + now=$(date +%s) + c=$(q "$CLIENT_EXPR" | awk '{print $2}') + o=$(q "$ORIGIN_EXPR" | awk '{print $2}') + h=$(q "$HITRATE_EXPR" | awk '{print $2}') + per=$(q "$PER_EXPR" | tr '\n' ' ') + [ -z "$c" ] && c=0 + [ -z "$o" ] && o=0 + [ -z "$h" ] && h=0 + line="$now ${c} ${o} ${h} $per" + echo "$line" + if [ -n "$OUT" ]; then echo "$line" >> "$OUT"; fi + sleep $STEP +done + +echo "# done" diff --git a/traffic-test/pip-traffic.yaml b/traffic-test/pip-traffic.yaml new file mode 100644 index 0000000..57bba3c --- /dev/null +++ b/traffic-test/pip-traffic.yaml @@ -0,0 +1,170 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-traffic-pip- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-traffic-pip +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 3600 + tasks: + - name: test-traffic-pip + replicas: 10 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + echo "== task: $(hostname) ==" + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -1 || true + apt-get install -y -qq git python3 python3-pip curl 2>&1 | tail -2 || { apt-get update -qq 2>&1 | tail -1 || true; apt-get install -y -qq git python3 python3-pip curl 2>&1 | tail -2; } + T_START=$(date +%s%3N) + + echo "" + echo "==========================================" + echo "1) git clone real repo (gitcode.com/Ascend/pytorch)" + echo "==========================================" + T0=$(date +%s%3N) + git clone --depth 1 --quiet https://gitcode.com/ascend/pytorch.git /workspace/pytorch + T1=$(date +%s%3N) + SIZE=$(du -sb /workspace/pytorch 2>/dev/null | awk '{print $1}') + echo "clone pytorch (depth1): $(( (T1 - T0) ))ms, size=${SIZE}B" + + echo "" + echo "==========================================" + echo "2) real requirements.txt from the repo" + echo "==========================================" + grep -vE '^\s*(#|$|--)' /workspace/pytorch/requirements.txt || true + + echo "" + echo "==========================================" + echo "3) bulk pip download (simulates 10 parallel CI installs)" + echo " real packages: repo requirements + common CI deps + torch 146MB wheel" + echo "==========================================" + mkdir -p /tmp/wheels + T0=$(date +%s%3N) + pip download --no-deps --no-cache-dir -v \ + --index-url https://repo.huaweicloud.com/repository/pypi/simple \ + -d /tmp/wheels \ + pyyaml setuptools auditwheel \ + numpy psutil requests tqdm regex pygments flask fastapi pydantic \ + torch==2.10.0 \ + 2>&1 | grep -oE "Saved .*" || true + T1=$(date +%s%3N) + BYTES=$(du -sb /tmp/wheels 2>/dev/null | awk '{print $1}') + COUNT=$(ls /tmp/wheels 2>/dev/null | wc -l) + MS=$((T1 - T0)) + echo "pip download: $COUNT files, ${BYTES:-0} bytes, ${MS}ms" + if [ "${BYTES:-0}" -gt 0 ] && [ "$MS" -gt 0 ]; then + echo "pip aggregate bandwidth: $(( BYTES * 1000 / MS )) B/s" + fi + + echo "" + echo "==========================================" + echo "4) pip install the small packages (real install phase)" + echo "==========================================" + T0=$(date +%s%3N) + pip install --quiet --break-system-packages --no-cache-dir \ + --index-url https://repo.huaweicloud.com/repository/pypi/simple \ + pyyaml setuptools auditwheel numpy psutil requests tqdm regex pygments flask fastapi pydantic + T1=$(date +%s%3N) + echo "pip install small pkgs: $((T1 - T0))ms" + + T_END=$(date +%s%3N) + echo "" + echo "DURATION: $((T_END - T_START))ms" + echo "== done $(hostname) ==" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then # RHEL/openEuler + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else # Debian/Ubuntu + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "%s";\n' \ + "$HTTPS_PROXY" "$HTTPS_PROXY" > /etc/apt/apt.conf.d/99squid-proxy + fi + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 3600 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh new file mode 100755 index 0000000..a45c410 --- /dev/null +++ b/traffic-test/run-tool-traffic.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# run-tool-traffic.sh — 10-parallel-task traffic test for the tool cases +# in squid-openssl/testcase/tool (cases 01..16), with per-case timestamps +# recorded for access.log window alignment. +# +# Usage: +# ./run-tool-traffic.sh 01 02 03 ... 16 # run selected cases (serial) +# ./run-tool-traffic.sh --all # run all 16 cases +# ./run-tool-traffic.sh --monitor 7200 # also sample squid traffic +# +# Logs: tracfic-test/logs/tool/-/.log +# Timeline: tracfic-test/logs/tool/timeline.tsv (ts case action job) +set -euo pipefail + +KUBECONFIG="${KUBECONFIG:-$HOME/.kube/gy-006.yaml}" +NS="squid" +DIR="$(cd "$(dirname "$0")" && pwd)" +TOOL_DIR="${TOOL_DIR:-$DIR/../../squid-openssl/testcase/tool}" +GEN="$DIR/gen-replicas.py" +GEN_DIR="$DIR/traffic-gen" +LOG_DIR="$DIR/logs/tool" +MONITOR=0 +MONITOR_DUR=7200 + +kc() { kubectl --kubeconfig "$KUBECONFIG" "$@"; } +tl() { echo -e "$(date +%s)\t$1\t$2\t$3" >> "$LOG_DIR/timeline.tsv"; } + +CASES=() +for a in "$@"; do + case "$a" in + --all) CASES=(01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16) ;; + --monitor) MONITOR=1 ;; + --monitor=*) MONITOR=1; MONITOR_DUR="${a#*=}" ;; + [0-9][0-9]) CASES+=("$a") ;; + *) echo "unknown arg: $a" >&2; exit 1 ;; + esac +done +[ ${#CASES[@]} -eq 0 ] && { echo "usage: $0 01 02 ... | --all [--monitor[=S]]" >&2; exit 1; } +for c in "${CASES[@]}"; do + ls "$TOOL_DIR/$c"*.yaml >/dev/null 2>&1 || { echo "case $c: yaml not found" >&2; exit 1; } +done + +mkdir -p "$GEN_DIR" "$LOG_DIR" +: > "$LOG_DIR/timeline.tsv" + +[ "$MONITOR" = "1" ] && { + echo "starting traffic monitor (${MONITOR_DUR}s)..." + "$DIR/monitor-traffic.sh" "$MONITOR_DUR" "$LOG_DIR/traffic.tsv" > "$LOG_DIR/traffic-monitor.log" 2>&1 & + MON_PID=$! +} + +submit_case() { # case-num → job + local num="$1" src gen + src=$(ls "$TOOL_DIR/$num"*.yaml | head -1) + gen="$GEN_DIR/$num-traffic.yaml" + python3 "$GEN" "$src" "$gen" 10 > /dev/null + local out jname + out=$(kc create -f "$gen" 2>&1) + jname=$(echo "$out" | grep -oE 'job\.batch\.volcano\.sh/[a-z0-9-]+' | tail -1 | cut -d/ -f2) + if [ -z "$jname" ]; then + echo "case $num: submit failed: $out" >&2 + tl "$num" SUBMIT_FAIL "$out" + return 1 + fi + echo "$jname" +} + +wait_case() { # job → rc + local job="$1" done=0 pod + local end=$(( $(date +%s) + 3600 )) + while [ "$(date +%s)" -lt "$end" ]; do + done=1 + for pod in $(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o name 2>/dev/null | cut -d/ -f2); do + case "$(kc get pod -n "$NS" "$pod" -o jsonpath='{.status.phase}' 2>/dev/null || echo Unknown)" in + Succeeded) ;; + Failed) echo " pod FAILED: $pod"; done=2 ;; + *) done=0 ;; + esac + done + [ "$done" = "1" ] && return 0 + [ "$done" = "2" ] && return 1 + sleep 10 + done + echo " TIMEOUT" >&2 + return 1 +} + +for c in "${CASES[@]}"; do + name=$(basename "$(ls "$TOOL_DIR/$c"*.yaml | head -1)" .yaml) + echo "== case $c ($name): submitting replicas=10 ==" + job=$(submit_case "$c") || { tl "$c" FAILED "-"; continue; } + tl "$c" SUBMIT "$job" + sleep 5 + if wait_case "$job"; then + sleep 5 + mkdir -p "$LOG_DIR/$c-$name" + i=0 + for pod in $(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o name 2>/dev/null | cut -d/ -f2); do + kc logs -n "$NS" "$pod" > "$LOG_DIR/$c-$name/pod-$i.log" 2>&1 || true + i=$((i+1)) + done + tl "$c" DONE "$job" + echo " done: $i pods → $LOG_DIR/$c-$name/" + else + tl "$c" FAILED "$job" + fi +done + +[ "$MONITOR" = "1" ] && { wait "$MON_PID" 2>/dev/null || true; } + +echo "" +echo "== timeline ==" +cat "$LOG_DIR/timeline.tsv" +echo "done" diff --git a/traffic-test/run.sh b/traffic-test/run.sh new file mode 100755 index 0000000..dc376ad --- /dev/null +++ b/traffic-test/run.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# run.sh — submit traffic test jobs to gy-006 (namespace squid) and wait. +# +# Usage: +# ./run.sh pip # pip traffic, 10 parallel tasks +# ./run.sh git # git clone traffic, 10 parallel tasks +# ./run.sh both # both, sequentially +# ./run.sh git --monitor 900 # also sample squid traffic from central Prometheus +# +# Kubeconfig: KUBECONFIG env or ~/.kube/gy-006.yaml +# Logs: tracfic-test/logs/-.log (one per pod) +set -euo pipefail + +KUBECONFIG="${KUBECONFIG:-$HOME/.kube/gy-006.yaml}" +NS="squid" +DIR="$(cd "$(dirname "$0")" && pwd)" +LOG_DIR="$DIR/logs" +MONITOR=0 +MONITOR_DUR=600 + +kc() { kubectl --kubeconfig "$KUBECONFIG" "$@"; } + +SUB="" +for a in "$@"; do + case "$a" in + pip|git|both) SUB="$a" ;; + --monitor) MONITOR=1 ;; + --monitor=*) MONITOR=1; MONITOR_DUR="${a#*=}" ;; + *) echo "unknown arg: $a" >&2; exit 1 ;; + esac +done +[ -z "$SUB" ] && { echo "usage: $0 pip|git|both [--monitor[=SECONDS]]" >&2; exit 1; } + +mkdir -p "$LOG_DIR" + +submit_wait() { # yaml → 0/1 (collects all pod logs) + local yaml="$1" name job pod + name=$(basename "${yaml%.yaml}") + echo "== submitting $name ==" + out=$(kc create -f "$yaml" 2>&1) + job=$(echo "$out" | grep -oE 'job\.batch\.volcano\.sh/[a-z0-9-]+' | tail -1 | cut -d/ -f2) + [ -z "$job" ] && { echo "submit failed: $out" >&2; return 1; } + echo " job: $job" + + local end=$(( $(date +%s) + 3600 )) + while [ "$(date +%s)" -lt "$end" ]; do + local done=1 + for pod in $(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o name 2>/dev/null | cut -d/ -f2); do + case "$(kc get pod -n "$NS" "$pod" -o jsonpath='{.status.phase}' 2>/dev/null || echo Unknown)" in + Succeeded) ;; + Failed) echo " POD FAILED: $pod"; done=2 ;; + *) done=0 ;; + esac + done + if [ "$done" = "1" ]; then + echo " all succeeded, collecting logs..." + sleep 5 + local i=0 + for pod in $(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o name 2>/dev/null | cut -d/ -f2); do + kc logs -n "$NS" "$pod" > "$LOG_DIR/$name-$job-$i.log" 2>&1 || true + i=$((i+1)) + done + echo " logs → $LOG_DIR/$name-$job-*.log" + return 0 + elif [ "$done" = "2" ]; then + kc logs -n "$NS" "$pod" > "$LOG_DIR/$name-$job-FAILED.log" 2>&1 || true + return 1 + fi + sleep 10 + done + echo " TIMEOUT waiting for $job" >&2 + return 1 +} + +[ "$MONITOR" = "1" ] && { + echo "starting traffic monitor (${MONITOR_DUR}s) in background..." + "$DIR/monitor-traffic.sh" "$MONITOR_DUR" "$LOG_DIR/traffic.tsv" > "$LOG_DIR/traffic-monitor.log" 2>&1 & + MON_PID=$! +} + +rc=0 +case "$SUB" in + pip) submit_wait "$DIR/pip-traffic.yaml" || rc=1 ;; + git) submit_wait "$DIR/git-clone-traffic.yaml" || rc=1 ;; + both) submit_wait "$DIR/pip-traffic.yaml" || rc=1 + submit_wait "$DIR/git-clone-traffic.yaml" || rc=1 ;; +esac + +[ "$MONITOR" = "1" ] && { wait "$MON_PID" 2>/dev/null || true; echo "traffic samples: $LOG_DIR/traffic.tsv"; } + +echo "" +echo "== summary (per-pod DURATION) ==" +for f in "$LOG_DIR"/"$name"*.log; do + [ -f "$f" ] || continue + d=$(grep -oE 'DURATION: [0-9]+ms' "$f" | head -1 | sed 's/DURATION: //') + t=$(basename "$f") + echo " $t → ${d:-n/a}" +done + +exit $rc From 90daca0c3375ecbcfe6cd0bb6e939448e4eb0849 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Thu, 13 Aug 2026 15:29:15 +0800 Subject: [PATCH 08/22] chore: rename branch tracfic-test -> traffic-test (typo fix) --- traffic-test/README.md | 2 +- traffic-test/RESULTS.md | 2 +- traffic-test/run-tool-traffic.sh | 4 ++-- traffic-test/run.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/traffic-test/README.md b/traffic-test/README.md index 542e08c..1671658 100644 --- a/traffic-test/README.md +++ b/traffic-test/README.md @@ -1,4 +1,4 @@ -# tracfic-test — 模拟 CI 并发流量的 Squid 流量模式测试 +# traffic-test — 模拟 CI 并发流量的 Squid 流量模式测试 在 gy-006 集群(namespace `squid`)用 **10 个并行 Volcano 任务**模拟真实 CI 并发, 通过 squid SSL-Bump 代理访问 `gitcode.com/Ascend` 真实仓库,观察 squid 的流量模式 diff --git a/traffic-test/RESULTS.md b/traffic-test/RESULTS.md index 2d4da9b..ecfda59 100644 --- a/traffic-test/RESULTS.md +++ b/traffic-test/RESULTS.md @@ -1,4 +1,4 @@ -# tracfic-test 实测结果(2026-08-12, gy-006) +# traffic-test 实测结果(2026-08-12, gy-006) 10 个并行 Volcano 任务模拟 CI 并发,全部走 squid SSL-Bump(`squid-cache.squid:3128`)。 数据来源:中央 Prometheus(`113.44.182.82:9090`,headless 每副本 scrape)+ 双副本 access.log。 diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh index a45c410..de0f3e9 100755 --- a/traffic-test/run-tool-traffic.sh +++ b/traffic-test/run-tool-traffic.sh @@ -8,8 +8,8 @@ # ./run-tool-traffic.sh --all # run all 16 cases # ./run-tool-traffic.sh --monitor 7200 # also sample squid traffic # -# Logs: tracfic-test/logs/tool/-/.log -# Timeline: tracfic-test/logs/tool/timeline.tsv (ts case action job) +# Logs: traffic-test/logs/tool/-/.log +# Timeline: traffic-test/logs/tool/timeline.tsv (ts case action job) set -euo pipefail KUBECONFIG="${KUBECONFIG:-$HOME/.kube/gy-006.yaml}" diff --git a/traffic-test/run.sh b/traffic-test/run.sh index dc376ad..0868430 100755 --- a/traffic-test/run.sh +++ b/traffic-test/run.sh @@ -8,7 +8,7 @@ # ./run.sh git --monitor 900 # also sample squid traffic from central Prometheus # # Kubeconfig: KUBECONFIG env or ~/.kube/gy-006.yaml -# Logs: tracfic-test/logs/-.log (one per pod) +# Logs: traffic-test/logs/-.log (one per pod) set -euo pipefail KUBECONFIG="${KUBECONFIG:-$HOME/.kube/gy-006.yaml}" From c0baf0cc26aae4df771f822a300d58229b731125 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Thu, 13 Aug 2026 11:45:05 +0800 Subject: [PATCH 09/22] feat(chart): distribute squid_bazel_trust_v3_jks via squid-ca-cert JKS truststore (Bazel/JDK) fetched from Vault (secrets/data/ascend/ci) and synced into squid-ca-cert SecretDefinition for CI namespaces (buildkitd, ascend-gha-runners, nv-action, squid). Align caBundleKey/ caPublicKey to v3 vault properties. --- deploy/chart/templates/secret-definition.yaml | 3 +++ deploy/chart/values.yaml | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/chart/templates/secret-definition.yaml b/deploy/chart/templates/secret-definition.yaml index c4b614e..95b99e0 100644 --- a/deploy/chart/templates/secret-definition.yaml +++ b/deploy/chart/templates/secret-definition.yaml @@ -27,5 +27,8 @@ spec: squid-ca.pem: path: {{ $.Values.secretDefinition.vaultPath }} key: {{ $.Values.secretDefinition.caPublicKey }} + {{ $.Values.secretDefinition.caTruststoreKey }}: + path: {{ $.Values.secretDefinition.vaultPath }} + key: {{ $.Values.secretDefinition.caTruststoreKey }} {{- end }} {{- end }} diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 91a3993..a3b08dc 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -88,8 +88,9 @@ persistence: secretDefinition: enabled: false vaultPath: secrets/data/ascend/ci - caBundleKey: squid_ca_bundle_pem - caPublicKey: squid_ca_pem + caBundleKey: squid_ca_bundle_v3_pem + caPublicKey: squid_ca_v3_pem + caTruststoreKey: squid_bazel_trust_v3_jks caNamespaces: [] # - buildkitd # - ascend-gha-runners From 3dd0a3e9c7ca281f531ba94c22b691c089b9b3bb Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Thu, 13 Aug 2026 11:46:32 +0800 Subject: [PATCH 10/22] refactor(chart): hardcode jks secret key name (like squid-ca.pem) --- deploy/chart/templates/secret-definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chart/templates/secret-definition.yaml b/deploy/chart/templates/secret-definition.yaml index 95b99e0..160504e 100644 --- a/deploy/chart/templates/secret-definition.yaml +++ b/deploy/chart/templates/secret-definition.yaml @@ -27,7 +27,7 @@ spec: squid-ca.pem: path: {{ $.Values.secretDefinition.vaultPath }} key: {{ $.Values.secretDefinition.caPublicKey }} - {{ $.Values.secretDefinition.caTruststoreKey }}: + squid_bazel_trust_v3_jks: path: {{ $.Values.secretDefinition.vaultPath }} key: {{ $.Values.secretDefinition.caTruststoreKey }} {{- end }} From 85cbe1b1edb9fb3e5b8eae47d09566ce1cb75664 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Thu, 13 Aug 2026 11:49:00 +0800 Subject: [PATCH 11/22] refactor(chart): use squid-bazel-trust.jks as jks secret key name --- deploy/chart/templates/secret-definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chart/templates/secret-definition.yaml b/deploy/chart/templates/secret-definition.yaml index 160504e..7dea0d0 100644 --- a/deploy/chart/templates/secret-definition.yaml +++ b/deploy/chart/templates/secret-definition.yaml @@ -27,7 +27,7 @@ spec: squid-ca.pem: path: {{ $.Values.secretDefinition.vaultPath }} key: {{ $.Values.secretDefinition.caPublicKey }} - squid_bazel_trust_v3_jks: + squid-bazel-trust.jks: path: {{ $.Values.secretDefinition.vaultPath }} key: {{ $.Values.secretDefinition.caTruststoreKey }} {{- end }} From efb301a25c98452d577b5f0042a2900d923f304d Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Thu, 13 Aug 2026 15:07:09 +0800 Subject: [PATCH 12/22] wip --- deploy/DEPLOY.md | 50 ++++++++++++++++++++++++++------------- deploy/chart/Chart.yaml | 2 +- deploy/tool/08-bazel.yaml | 28 ++++++++++++++++------ tracfic-test/.gitignore | 2 ++ 4 files changed, 58 insertions(+), 24 deletions(-) create mode 100644 tracfic-test/.gitignore diff --git a/deploy/DEPLOY.md b/deploy/DEPLOY.md index 5a85706..e2dde50 100644 --- a/deploy/DEPLOY.md +++ b/deploy/DEPLOY.md @@ -51,8 +51,9 @@ Squid SSL-Bump 需要一套 CA。**出于安全,CA 私有材料不在本仓库 | `squid-ca` | `squid` | `squid-ca-bundle.pem` | **私钥+证书**(SSL-Bump 签发) | StatefulSet 的 squid 容器(挂到 `/etc/squid/ssl_cert/`),init 容器再拆出 registry-proxy 的 ca.crt/ca.key | | `squid-ca` | `squid` | `squid-ca.pem` | 公钥证书 | 备用 | | `squid-ca-cert` | 每个用代理的 ns | `squid-ca.pem` | **仅公钥证书** | CI 客户端信任链(挂到 `/etc/squid-ca/`,见 §2.3) | +| `squid-ca-cert` | 每个用代理的 ns | `squid-bazel-trust.jks` | **JKS trust store**(squid CA + 系统根,bazel/JVM 专用,见 §2.5) | bazel 构建(挂到 `/etc/squid-bazel-trust/`) | -同步行为由 `values.secretDefinition`(`enabled` / `vaultPath` / `caBundleKey` / `caPublicKey` / `caNamespaces`)控制。 +同步行为由 `values.secretDefinition`(`enabled` / `vaultPath` / `caBundleKey` / `caPublicKey` / `caTruststoreKey` / `caNamespaces`)控制。 生产集群(如 gy-006)走这条路径,**无需手动建 secret**——只要保证引用的 secret name 与上表一致即可。 > 下面的 `kubectl create secret` 仅为**无 Vault 的临时/测试集群**的回退手段(明文操作 CA,切勿用于生产)。 @@ -71,19 +72,20 @@ kubectl -n squid create secret generic squid-ca \ --from-file=squid-ca.pem=../squid-openssl/ca/006-ca-new/squid-ca.pem \ --dry-run=client -o yaml | kubectl apply -f - -# 2) CI 命名空间用的 CA 公钥(每个需要代理的命名空间各一份) +# 2) CI 命名空间用的 CA 公钥 + bazel JKS(每个需要代理的命名空间各一份) +# JKS 生成方式:keytool -importcert -alias squid-ca -file squid-ca.pem \ +# -keystore squid-bazel-trust.jks -storepass changeit -noprompt kubectl -n squid create secret generic squid-ca-cert \ --from-file=squid-ca.pem=../squid-openssl/ca/006-ca-new/squid-ca.pem \ - --dry-run=client -o yaml | kubectl apply -f - - -# 3) Bazel JVM trust store(bazel 客户端专用, 见 §2.5) -kubectl -n squid create configmap squid-bazel-trust \ --from-file=squid-bazel-trust.jks=../squid-openssl/ca/006-ca-new/squid-bazel-trust.jks \ --dry-run=client -o yaml | kubectl apply -f - ``` > 若配置了 `secretDefinition.enabled: true`,`squid-ca` 与各命名空间的 `squid-ca-cert` -> 由 secrets-manager 从 Vault 自动同步,无需手动创建。 +> (含 `squid-bazel-trust.jks`)由 secrets-manager 从 Vault 自动同步,无需手动创建。 +> ⚠️ **Vault 只存字符串**:JKS 二进制写入 Vault 时会自动 base64 编码(CLI/API 对非 UTF-8 +> 内容统一 base64)。因此挂载出来的 `squid-bazel-trust.jks` 是 base64 文本,消费端需解码 +> 一次(见 §2.5 的 postStart 配方),这是 Vault 存二进制的标准形态。 ### 1.3 ArgoCD 部署 @@ -200,9 +202,6 @@ volumeMounts: - name: squid-ca mountPath: /etc/squid-ca readOnly: true -- name: squid-bazel-trust # 仅 bazel/Java 构建需要 - mountPath: /etc/squid-bazel-trust - readOnly: true volumes: - name: squid-ca secret: @@ -210,13 +209,14 @@ volumes: items: - key: squid-ca.pem path: squid-ca.pem - optional: true -- name: squid-bazel-trust - configMap: - name: squid-bazel-trust + - key: squid-bazel-trust.jks # Vault 同步的 base64 JKS(见 §1.2) + path: squid-bazel-trust.jks optional: true ``` +> ⚠️ K8s 的 secret/configmap 卷**无论 manifest 是否写 `readOnly` 都是只读的**(kubelet 投影, +> CRI 挂载固定 `ro`)。解码产物写到容器层普通目录(如 `/etc/squid-bazel-trust`,`mkdir -p` 即可)或 `/tmp`。 + ### 2.4 postStart 钩子(把 CA 装进系统信任库 + 工具专项) ```yaml @@ -251,17 +251,35 @@ lifecycle: ### 2.5 Bazel 专项(JVM trust store) -Bazel 启动 JVM 时**忽略 `JAVA_TOOL_OPTIONS`**,trust 参数只能通过 `.bazelrc`: +Bazel 启动 JVM 时**忽略 `JAVA_TOOL_OPTIONS`**,trust 参数只能通过 `.bazelrc`。 +JKS 经 Vault 同步后是 **base64 文本**(Vault 只存字符串,二进制自动 base64), +postStart 需先解码再写 `.bazelrc`: ```yaml # postStart 中: +S=/etc/squid-ca/squid-bazel-trust.jks # secret 只读层(base64 文本) +J=/etc/squid-bazel-trust/squid-bazel-trust.jks # 容器层普通目录(真 JKS) +mkdir -p /etc/squid-bazel-trust +if [ -f "$S" ]; then + if base64 -d "$S" > "$J" 2>/dev/null \ + && [ -s "$J" ] \ + && [ "$(od -An -tx1 -N4 "$J" | tr -d ' ')" = "feedfeed" ]; then + echo "JKS decoded from base64 -> $J" + else + cp "$S" "$J" 2>/dev/null # 已是原始二进制则直接拷贝 + echo "JKS used as-is -> $J" + fi +fi cat > "$WORKSPACE/.bazelrc" << 'EOF' startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit EOF ``` -- jks 由 `squid-bazel-trust` ConfigMap 提供(生成方式:`keytool -importcert -alias squid-ca -file squid-ca.pem -keystore squid-bazel-trust.jks -storepass changeit -noprompt`)。 +- JKS 由 Vault 经 `squid-ca-cert` secret 分发(`key: squid-bazel-trust.jks`),生成方式: + `keytool -importcert -alias squid-ca -file squid-ca.pem -keystore squid-bazel-trust.jks -storepass changeit -noprompt` + (把 squid CA 导入**系统根**副本,`-storepass` 默认 `changeit`)。 +- ⚠️ 坑:secret 卷只读,解码必须写到容器层目录(`mkdir -p /etc/squid-bazel-trust`)或 `/tmp`(实测写 secret 挂载点会 EROFS 静默失败,JVM 回退默认信任库 → PKIX)。 - github.com 下载超时场景:用 gh-proxy(`https://gh-proxy.test.osinfra.cn/https://github.com/...`)替换 URL,见 `tool/08-bazel.yaml` 的 WORKSPACE 写法。 ### 2.6 完整模板 diff --git a/deploy/chart/Chart.yaml b/deploy/chart/Chart.yaml index bd347fa..5cd3d03 100644 --- a/deploy/chart/Chart.yaml +++ b/deploy/chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: squid-rpardini description: Squid proxy with SSL Bump + registry cache sidecar for CI workloads type: application -version: 0.1.4 +version: 0.1.5 appVersion: "7.6" diff --git a/deploy/tool/08-bazel.yaml b/deploy/tool/08-bazel.yaml index ee16c64..cc83a6a 100644 --- a/deploy/tool/08-bazel.yaml +++ b/deploy/tool/08-bazel.yaml @@ -117,6 +117,25 @@ spec: - | set +e + # JKS is stored base64-encoded in Vault (KV v2 stores + # strings only) → decode to a real keystore on disk. + # The secret mount is read-only, but /etc/squid-bazel-trust + # is a plain writable dir in the container layer. + S=/etc/squid-ca/squid-bazel-trust.jks + J=/etc/squid-bazel-trust/squid-bazel-trust.jks + mkdir -p /etc/squid-bazel-trust + if [ -f "$S" ]; then + if base64 -d "$S" > "$J" 2>/dev/null \ + && [ -s "$J" ] \ + && [ "$(od -An -tx1 -N4 "$J" | tr -d ' ')" = "feedfeed" ]; then + echo "JKS decoded from base64 -> $J" + else + cp "$S" "$J" 2>/dev/null + echo "JKS used as-is -> $J" + fi + chmod 644 "$J" 2>/dev/null + fi + # Bazel JVM trust: point JVM at the mounted squid-CA keystore # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) cat > "$WORKSPACE/.bazelrc" << 'EOF' @@ -180,9 +199,6 @@ spec: - name: squid-ca mountPath: /etc/squid-ca readOnly: true - - name: squid-bazel-trust - mountPath: /etc/squid-bazel-trust - readOnly: true volumes: - name: squid-ca secret: @@ -190,10 +206,8 @@ spec: items: - key: squid-ca.pem path: squid-ca.pem - optional: true - - name: squid-bazel-trust - configMap: - name: squid-bazel-trust + - key: squid-bazel-trust.jks + path: squid-bazel-trust.jks optional: true nodeSelector: kubernetes.io/arch: arm64 diff --git a/tracfic-test/.gitignore b/tracfic-test/.gitignore new file mode 100644 index 0000000..683daf1 --- /dev/null +++ b/tracfic-test/.gitignore @@ -0,0 +1,2 @@ +logs +traffic-gen \ No newline at end of file From ebd67731bbb370530f7f0cb18545f819fac3a4d5 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Thu, 13 Aug 2026 15:44:20 +0800 Subject: [PATCH 13/22] wip --- tracfic-test/.gitignore | 2 - {deploy => traffic-test}/tool/.gen-direct.py | 0 {deploy => traffic-test}/tool/.gitignore | 0 {deploy => traffic-test}/tool/01-pip.yaml | 0 {deploy => traffic-test}/tool/02-apt.yaml | 0 {deploy => traffic-test}/tool/03-github.yaml | 0 {deploy => traffic-test}/tool/04-goproxy.yaml | 0 {deploy => traffic-test}/tool/05-obs.yaml | 0 {deploy => traffic-test}/tool/06-wget.yaml | 0 .../tool/07-cmake-fetchcontent.yaml | 0 {deploy => traffic-test}/tool/08-bazel.yaml | 0 {deploy => traffic-test}/tool/09-npm.yaml | 0 traffic-test/tool/10-cargo.yaml | 180 ++++++++++++++++++ traffic-test/tool/11-conda.yaml | 166 ++++++++++++++++ traffic-test/tool/12-uv.yaml | 157 +++++++++++++++ traffic-test/tool/13-huggingface.yaml | 168 ++++++++++++++++ traffic-test/tool/14-gitlfs.yaml | 170 +++++++++++++++++ traffic-test/tool/15-pnpm.yaml | 165 ++++++++++++++++ traffic-test/tool/16-yum.yaml | 159 ++++++++++++++++ {deploy => traffic-test}/tool/FINAL-REPORT.md | 0 {deploy => traffic-test}/tool/SUMMARY.txt | 0 .../tool/ascend-org-build-tools-report.md | 0 {deploy => traffic-test}/tool/cachedemo.yaml | 0 {deploy => traffic-test}/tool/eval-logs.sh | 0 .../tool/harvest-results.sh | 0 .../tool/run-tool-tests.sh | 0 26 files changed, 1165 insertions(+), 2 deletions(-) delete mode 100644 tracfic-test/.gitignore rename {deploy => traffic-test}/tool/.gen-direct.py (100%) rename {deploy => traffic-test}/tool/.gitignore (100%) rename {deploy => traffic-test}/tool/01-pip.yaml (100%) rename {deploy => traffic-test}/tool/02-apt.yaml (100%) rename {deploy => traffic-test}/tool/03-github.yaml (100%) rename {deploy => traffic-test}/tool/04-goproxy.yaml (100%) rename {deploy => traffic-test}/tool/05-obs.yaml (100%) rename {deploy => traffic-test}/tool/06-wget.yaml (100%) rename {deploy => traffic-test}/tool/07-cmake-fetchcontent.yaml (100%) rename {deploy => traffic-test}/tool/08-bazel.yaml (100%) rename {deploy => traffic-test}/tool/09-npm.yaml (100%) create mode 100644 traffic-test/tool/10-cargo.yaml create mode 100644 traffic-test/tool/11-conda.yaml create mode 100644 traffic-test/tool/12-uv.yaml create mode 100644 traffic-test/tool/13-huggingface.yaml create mode 100644 traffic-test/tool/14-gitlfs.yaml create mode 100644 traffic-test/tool/15-pnpm.yaml create mode 100644 traffic-test/tool/16-yum.yaml rename {deploy => traffic-test}/tool/FINAL-REPORT.md (100%) rename {deploy => traffic-test}/tool/SUMMARY.txt (100%) rename {deploy => traffic-test}/tool/ascend-org-build-tools-report.md (100%) rename {deploy => traffic-test}/tool/cachedemo.yaml (100%) rename {deploy => traffic-test}/tool/eval-logs.sh (100%) rename {deploy => traffic-test}/tool/harvest-results.sh (100%) rename {deploy => traffic-test}/tool/run-tool-tests.sh (100%) diff --git a/tracfic-test/.gitignore b/tracfic-test/.gitignore deleted file mode 100644 index 683daf1..0000000 --- a/tracfic-test/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -logs -traffic-gen \ No newline at end of file diff --git a/deploy/tool/.gen-direct.py b/traffic-test/tool/.gen-direct.py similarity index 100% rename from deploy/tool/.gen-direct.py rename to traffic-test/tool/.gen-direct.py diff --git a/deploy/tool/.gitignore b/traffic-test/tool/.gitignore similarity index 100% rename from deploy/tool/.gitignore rename to traffic-test/tool/.gitignore diff --git a/deploy/tool/01-pip.yaml b/traffic-test/tool/01-pip.yaml similarity index 100% rename from deploy/tool/01-pip.yaml rename to traffic-test/tool/01-pip.yaml diff --git a/deploy/tool/02-apt.yaml b/traffic-test/tool/02-apt.yaml similarity index 100% rename from deploy/tool/02-apt.yaml rename to traffic-test/tool/02-apt.yaml diff --git a/deploy/tool/03-github.yaml b/traffic-test/tool/03-github.yaml similarity index 100% rename from deploy/tool/03-github.yaml rename to traffic-test/tool/03-github.yaml diff --git a/deploy/tool/04-goproxy.yaml b/traffic-test/tool/04-goproxy.yaml similarity index 100% rename from deploy/tool/04-goproxy.yaml rename to traffic-test/tool/04-goproxy.yaml diff --git a/deploy/tool/05-obs.yaml b/traffic-test/tool/05-obs.yaml similarity index 100% rename from deploy/tool/05-obs.yaml rename to traffic-test/tool/05-obs.yaml diff --git a/deploy/tool/06-wget.yaml b/traffic-test/tool/06-wget.yaml similarity index 100% rename from deploy/tool/06-wget.yaml rename to traffic-test/tool/06-wget.yaml diff --git a/deploy/tool/07-cmake-fetchcontent.yaml b/traffic-test/tool/07-cmake-fetchcontent.yaml similarity index 100% rename from deploy/tool/07-cmake-fetchcontent.yaml rename to traffic-test/tool/07-cmake-fetchcontent.yaml diff --git a/deploy/tool/08-bazel.yaml b/traffic-test/tool/08-bazel.yaml similarity index 100% rename from deploy/tool/08-bazel.yaml rename to traffic-test/tool/08-bazel.yaml diff --git a/deploy/tool/09-npm.yaml b/traffic-test/tool/09-npm.yaml similarity index 100% rename from deploy/tool/09-npm.yaml rename to traffic-test/tool/09-npm.yaml diff --git a/traffic-test/tool/10-cargo.yaml b/traffic-test/tool/10-cargo.yaml new file mode 100644 index 0000000..af7afe9 --- /dev/null +++ b/traffic-test/tool/10-cargo.yaml @@ -0,0 +1,180 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-cargo- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-cargo +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-cargo + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -2 || true + apt-get install -y -qq cargo rustc 2>&1 | tail -3 || { apt-get update -qq 2>&1 | tail -2 || true; apt-get install -y -qq cargo rustc 2>&1 | tail -3; } + T_START=$(date +%s%3N) + + # rustls builds don't read the system store, so CARGO_HTTP_CAINFO + # (pointing at the mounted squid CA) comes from the pod env + cargo --version; rustc --version + + echo "" + echo "==========================================" + echo "Scenario 10: cargo fetch (crates.io / static.crates.io —" + echo " text-embeddings-inference, MindIE-Motor pattern)" + echo "==========================================" + + # crates.io has 2 endpoints; both go through squid: + # static.crates.io/crates//-.crate (binary) + # index.crates.io (sparse index, JSON) + # Force the sparse index (default since 1.70) and pin a registry. + mkdir -p /workspace/app && cd /workspace/app + cargo init --name hello 2>&1 | tail -1 + + # rsproxy.cn (npmmirror team) sparse mirror for crates.io + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "rsproxy-sparse" + + [source.rsproxy-sparse] + registry = "sparse+https://rsproxy.cn/index/" + EOF + cat >> Cargo.toml << 'EOF' + anyhow = "1" + serde_json = "1" + tokio = { version = "1", features = ["rt-multi-thread"] } + EOF + + echo "--- cargo fetch (crates.io via squid, cold) ---" + T0=$(date +%s%3N) + cargo fetch 2>&1 | tail -3 + T1=$(date +%s%3N) + echo "fetch: $((T1 - T0))ms" + + echo "" + echo "--- quick build to prove the deps resolve ---" + cargo build 2>&1 | tail -2 + echo "✅ cargo fetch + build through squid OK" + + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 10 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + - name: CARGO_HTTP_CAINFO + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/tool/11-conda.yaml b/traffic-test/tool/11-conda.yaml new file mode 100644 index 0000000..01c1388 --- /dev/null +++ b/traffic-test/tool/11-conda.yaml @@ -0,0 +1,166 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-conda- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-conda +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-conda + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -2 || true + apt-get install -y -qq wget curl 2>&1 | tail -3 || { apt-get update -qq 2>&1 | tail -2 || true; apt-get install -y -qq wget curl 2>&1 | tail -3; } + T_START=$(date +%s%3N) + + # conda (python/requests-based) honors https_proxy + SSL_CERT_FILE + + echo "" + echo "==========================================" + echo "Scenario 11: conda (conda-forge — DrivingSDK, model-agent," + echo " MindSpeed-MM, modelzoo-GPL ...)" + echo "==========================================" + + echo "--- Download Miniconda installer (arm64, ~100MB via squid) ---" + cd /tmp + T0=$(date +%s%3N) + wget -q --timeout=90 --tries=2 https://mirror.nju.edu.cn/anaconda/miniconda/Miniconda3-latest-Linux-aarch64.sh -O miniconda.sh + T1=$(date +%s%3N) + echo "installer download: $((T1 - T0))ms ($(stat -c%s miniconda.sh) bytes)" + + echo "" + echo "--- Install miniconda ---" + bash miniconda.sh -b -p /opt/conda > /dev/null 2>&1 + export PATH=/opt/conda/bin:$PATH + conda --version + + echo "" + echo "--- conda create python+numpy from conda-forge (cold) ---" + T0=$(date +%s%3N) + conda create -y -n testenv --override-channels -c https://mirror.nju.edu.cn/anaconda/cloud/conda-forge/ python=3.11 numpy 2>&1 | tail -6 + T1=$(date +%s%3N) + echo "conda create: $((T1 - T0))ms" + + echo "" + conda run -n testenv python -c "import numpy; print('✅ numpy', numpy.__version__, 'works through squid')" + + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 11 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/tool/12-uv.yaml b/traffic-test/tool/12-uv.yaml new file mode 100644 index 0000000..65fc034 --- /dev/null +++ b/traffic-test/tool/12-uv.yaml @@ -0,0 +1,157 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-uv- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-uv +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-uv + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -2 || true + apt-get install -y -qq python3 python3-pip 2>&1 | tail -3 || { apt-get update -qq 2>&1 | tail -2 || true; apt-get install -y -qq python3 python3-pip 2>&1 | tail -3; } + T_START=$(date +%s%3N) + export UV_DEFAULT_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/simple + echo "" + echo "==========================================" + echo "Scenario 12: uv (uv sync / uv pip — msmodeling's package manager)" + echo "==========================================" + + echo "--- Install uv itself (a pip wheel via squid) ---" + pip install -q --break-system-packages -i https://mirrors.huaweicloud.com/repository/pypi/simple uv 2>&1 | tail -1 + uv --version + + echo "" + echo "--- uv venv + uv pip install numpy requests (cold) ---" + mkdir -p /workspace/uvproj && cd /workspace/uvproj + uv venv /workspace/uvproj/.venv 2>&1 | tail -1 + T0=$(date +%s%3N) + uv pip install --python /workspace/uvproj/.venv/bin/python numpy requests 2>&1 | tail -3 + T1=$(date +%s%3N) + echo "fetch: $((T1 - T0))ms" + + echo "" + echo "" + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 12 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + - name: UV_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/tool/13-huggingface.yaml b/traffic-test/tool/13-huggingface.yaml new file mode 100644 index 0000000..047d3e2 --- /dev/null +++ b/traffic-test/tool/13-huggingface.yaml @@ -0,0 +1,168 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-hf- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-hf +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-hf + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -2 || true + apt-get install -y -qq python3 python3-pip 2>&1 | tail -3 || { apt-get update -qq 2>&1 | tail -2 || true; apt-get install -y -qq python3 python3-pip 2>&1 | tail -3; } + T_START=$(date +%s%3N) + export HF_ENDPOINT=https://hf-mirror.com + echo "" + echo "==========================================" + echo "Scenario 13: huggingface_hub (snapshot_download / hf download —" + echo " 328 files across pytorch, MindSpeed-MM, DrivingSDK...)" + echo "==========================================" + + echo "--- Install huggingface_hub (pip wheel via squid) ---" + pip install -q --break-system-packages -i https://mirrors.huaweicloud.com/repository/pypi/simple huggingface_hub 2>&1 | tail -1 + hf --version + + # tiny-gpt2: config ~2MB + LFS-weights pytorch_model.bin (2.5MB) — + # the model-download pattern from pytorch/benchmarks/llm/download_hf.py + MODEL="sshleifer/tiny-gpt2" + + echo "" + echo "--- hf download (cold, via cdn-lfs.huggingface.co) ---" + T0=$(date +%s%3N) + hf download "$MODEL" --local-dir /tmp/hf 2>&1 | tail -4 + T1=$(date +%s%3N) + echo "fetch: $((T1 - T0))ms, $(du -sh /tmp/hf | cut -f1)" + + echo "" + echo "" + echo "--- python API check (snapshot_download — the actual org pattern) ---" + python3 - << 'EOF' + from huggingface_hub import snapshot_download + p = snapshot_download("sshleifer/tiny-gpt2", local_dir="/tmp/hf-snap") + import os + print("✅ snapshot_download OK:", p, "| files:", len(os.listdir(p))) + EOF + + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 13 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "0" + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/tool/14-gitlfs.yaml b/traffic-test/tool/14-gitlfs.yaml new file mode 100644 index 0000000..d64c587 --- /dev/null +++ b/traffic-test/tool/14-gitlfs.yaml @@ -0,0 +1,170 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-gitlfs- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-gitlfs +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-gitlfs + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -2 || true + apt-get install -y -qq git curl 2>&1 | tail -3 || { apt-get update -qq 2>&1 | tail -2 || true; apt-get install -y -qq git curl 2>&1 | tail -3; } + T_START=$(date +%s%3N) + echo "" + echo "==========================================" + echo "Scenario 14: git-lfs (model files in LFS — ascend-docker-image," + echo " DrivingSDK GR00T, mind-cluster docs)" + echo "==========================================" + + echo "--- Install git-lfs (github release binary via squid) ---" + curl -sL -o /tmp/git-lfs.tar.gz \ + https://gh-proxy.test.osinfra.cn/https://github.com/git-lfs/git-lfs/releases/download/v3.6.1/git-lfs-linux-arm64-v3.6.1.tar.gz + tar -C /usr/local -xzf /tmp/git-lfs.tar.gz 2>&1 | tail -1 + export PATH=/usr/local/git-lfs-3.6.1:$PATH + git lfs version + git lfs install --skip-repo 2>&1 | tail -1 + + echo "" + echo "--- clone GitHub LFS test repo (git-lfs/git-lfs) via gh-proxy ---" + # git-lfs/git-lfs has test fixtures stored as LFS objects + rm -rf /workspace/lfstest && mkdir -p /workspace/lfstest && cd /workspace/lfstest + T0=$(date +%s%3N) + GIT_LFS_SKIP_SMUDGE=0 git clone --depth=1 https://gh-proxy.test.osinfra.cn/https://github.com/git-lfs/git-lfs repo 2>&1 | tail -3 + T1=$(date +%s%3N) + echo "clone: $((T1 - T0))ms" + echo "LFS files:" + git -C repo lfs ls-files 2>&1 | head -5 + du -sh repo + + echo "" + echo "" + echo "--- git lfs pull round-trip (fetch-only mode) ---" + cd repo + T0=$(date +%s%3N) + git lfs pull 2>&1 | tail -2 + T1=$(date +%s%3N) + echo "git lfs pull: $((T1 - T0))ms" + echo "✅ git-lfs objects downloaded through squid" + + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 14 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/tool/15-pnpm.yaml b/traffic-test/tool/15-pnpm.yaml new file mode 100644 index 0000000..63ac7a6 --- /dev/null +++ b/traffic-test/tool/15-pnpm.yaml @@ -0,0 +1,165 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-pnpm- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-pnpm +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-pnpm + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ubuntu:24.04 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ + /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true + apt-get update -qq 2>&1 | tail -2 || true + apt-get install -y -qq nodejs npm 2>&1 | tail -3 || { apt-get update -qq 2>&1 | tail -2 || true; apt-get install -y -qq nodejs npm 2>&1 | tail -3; } + T_START=$(date +%s%3N) + echo "" + echo "==========================================" + echo "Scenario 15: pnpm (AgentSDK/openclaw uses pnpm — openclaw" + echo " install_to_image.sh / install-sc-local.sh)" + echo "==========================================" + + echo "--- Install pnpm via corepack (node bundled) ---" + npm config set registry https://registry.npmmirror.com/ + corepack enable 2>&1 | tail -1 || true + npm install -g pnpm@9 2>&1 | tail -2 + pnpm --version + + pnpm config set registry https://registry.npmmirror.com/ + echo "pnpm registry: $(pnpm config get registry)" + + mkdir -p /workspace/papp && cd /workspace/papp + cat > package.json << 'EOF' + { "name": "pnpm-test", "version": "1.0.0", "private": true } + EOF + + echo "" + echo "--- pnpm install express (cold, via npmmirror) ---" + T0=$(date +%s%3N) + pnpm add express --no-optional 2>&1 | tail -4 + T1=$(date +%s%3N) + echo "fetch: $((T1 - T0))ms" + ls node_modules/.pnpm | wc -l + + echo "" + node -e "require('express'); console.log('✅ pnpm-installed express works through squid')" + + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 15 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/traffic-test/tool/16-yum.yaml b/traffic-test/tool/16-yum.yaml new file mode 100644 index 0000000..9cd38aa --- /dev/null +++ b/traffic-test/tool/16-yum.yaml @@ -0,0 +1,159 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + generateName: test-squid-yum- + namespace: squid + labels: + kubernetes.io/arch: arm64 + pipeline/run-id: test-squid-yum +spec: + policies: + - event: PodFailed + action: AbortJob + queue: shared-flexible-queue + maxRetry: 1 + minAvailable: 1 + ttlSecondsAfterFinished: 1800 + tasks: + - name: test-squid-yum + replicas: 1 + maxRetry: 1 + template: + spec: + containers: + - name: test + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/openeuler/openeuler:24.03 + command: + - bash + - -c + args: + - | + #!/bin/bash + set -e + T_START=$(date +%s%3N) + + echo "" + echo "==========================================" + echo "Scenario 16: yum/dnf install (openEuler 24.03)" + echo "==========================================" + echo "os: $(grep -E '^(NAME|VERSION)=' /etc/os-release | tr '\n' ' ')" + echo "yum: $(yum --version 2>&1 | head -1)" + echo "--- switch yum repos to huaweicloud mirror ---" + sed -i 's|https://repo.openeuler.org|https://mirrors.huaweicloud.com/openeuler|g; /^metalink=/d' /etc/yum.repos.d/*.repo 2>/dev/null || true + grep -h "baseurl" /etc/yum.repos.d/openEuler.repo 2>/dev/null | head -1 + + echo "" + echo "--- yum makecache (repo metadata via squid) ---" + T0=$(date +%s%3N) + timeout 120 yum makecache 2>&1 | tail -3 + T1=$(date +%s%3N) + echo "makecache: $((T1 - T0))ms" + + echo "" + echo "--- yum install packages (binary RPMs via squid) ---" + T0=$(date +%s%3N) + timeout 120 yum install -y curl wget git jq 2>&1 | tail -3 + T1=$(date +%s%3N) + echo "yum install: $((T1 - T0))ms" + curl --version | head -1 + wget --version | head -1 + git --version + jq --version + + echo "" + echo "==========================================" + T_END=$(date +%s%3N) + echo "DURATION: $((T_END - T_START))ms" + echo "✅ Scenario 16 completed." + echo "==========================================" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set +e + + # Bazel JVM trust: point JVM at the mounted squid-CA keystore + # (Bazel ignores JAVA_TOOL_OPTIONS; args must come via .bazelrc) + cat > "$WORKSPACE/.bazelrc" << 'EOF' + common --noenable_bzlmod + common --registry=https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/ + build --cxxopt=-std=c++17 + startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/squid-bazel-trust/squid-bazel-trust.jks + startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit + EOF + + P=/etc/squid-ca/squid-ca.pem + if [ -f "$P" ]; then + if [ -d /etc/pki/ca-trust/source/anchors ]; then + cp "$P" /etc/pki/ca-trust/source/anchors/squid-ca.pem >/dev/null 2>&1 + update-ca-trust extract >/dev/null 2>&1 + else + cp "$P" /usr/local/share/ca-certificates/squid-ca.crt >/dev/null 2>&1 + update-ca-certificates -f >/dev/null 2>&1 + fi + fi + + if command -v apt-get >/dev/null 2>&1 && [ -n "$HTTPS_PROXY" ]; then + mkdir -p /etc/apt/apt.conf.d + printf 'Acquire::http::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\nAcquire::https::Proxy "http://squid-cache.squid.svc.cluster.local:3128";\n' > /etc/apt/apt.conf.d/99squid-proxy + fi + + exit 0 + workingDir: /workspace + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + env: + - name: WORKSPACE + value: /workspace + - name: HTTP_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: HTTPS_PROXY + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: http_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: https_proxy + value: "http://squid-cache.squid.svc.cluster.local:3128" + - name: NO_PROXY + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: no_proxy + value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + - name: SSL_CERT_FILE + value: /etc/squid-ca/squid-ca.pem + - name: CURL_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: REQUESTS_CA_BUNDLE + value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem + - name: PIP_CERT + value: /etc/squid-ca/squid-ca.pem + - name: NODE_EXTRA_CA_CERTS + value: /etc/squid-ca/squid-ca.pem + volumeMounts: + - name: squid-ca + mountPath: /etc/squid-ca + readOnly: true + volumes: + - name: squid-ca + secret: + secretName: squid-ca-cert + items: + - key: squid-ca.pem + path: squid-ca.pem + optional: true + nodeSelector: + kubernetes.io/arch: arm64 + imagePullSecrets: + - name: huawei-swr-image-pull-secret-model-gy + activeDeadlineSeconds: 2400 + securityContext: + runAsUser: 0 + restartPolicy: Never diff --git a/deploy/tool/FINAL-REPORT.md b/traffic-test/tool/FINAL-REPORT.md similarity index 100% rename from deploy/tool/FINAL-REPORT.md rename to traffic-test/tool/FINAL-REPORT.md diff --git a/deploy/tool/SUMMARY.txt b/traffic-test/tool/SUMMARY.txt similarity index 100% rename from deploy/tool/SUMMARY.txt rename to traffic-test/tool/SUMMARY.txt diff --git a/deploy/tool/ascend-org-build-tools-report.md b/traffic-test/tool/ascend-org-build-tools-report.md similarity index 100% rename from deploy/tool/ascend-org-build-tools-report.md rename to traffic-test/tool/ascend-org-build-tools-report.md diff --git a/deploy/tool/cachedemo.yaml b/traffic-test/tool/cachedemo.yaml similarity index 100% rename from deploy/tool/cachedemo.yaml rename to traffic-test/tool/cachedemo.yaml diff --git a/deploy/tool/eval-logs.sh b/traffic-test/tool/eval-logs.sh similarity index 100% rename from deploy/tool/eval-logs.sh rename to traffic-test/tool/eval-logs.sh diff --git a/deploy/tool/harvest-results.sh b/traffic-test/tool/harvest-results.sh similarity index 100% rename from deploy/tool/harvest-results.sh rename to traffic-test/tool/harvest-results.sh diff --git a/deploy/tool/run-tool-tests.sh b/traffic-test/tool/run-tool-tests.sh similarity index 100% rename from deploy/tool/run-tool-tests.sh rename to traffic-test/tool/run-tool-tests.sh From df0a3c44cc51dbe288c96bede36e8766cb4f77a4 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Fri, 14 Aug 2026 11:04:43 +0800 Subject: [PATCH 14/22] =?UTF-8?q?feat(chart):=20cache=20strategy=200.1.6?= =?UTF-8?q?=20=E2=80=94=20immutable=20by=20addressing,=20drop=20invalid=20?= =?UTF-8?q?options?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - immutable rules only for content-addressed/A-class artifacts (.whl/.crate/.deb) - .zip/.tar.gz/.pth demoted to LM-aging (0/20%/4320): ref-addressed & overwritable - codeload refs/tags + github releases/download get immutable; refs/heads short TTL - remove dead .docker.io rule, unescape domains, max 525960->525600 (no cropped warning) - add CACHE-STRATEGY.md (full per-rule audit + CI scenario analysis) --- deploy/CACHE-STRATEGY.md | 219 ++++++++++++++++++++++++++ deploy/SQUID-OVERVIEW.md | 10 +- deploy/chart/Chart.yaml | 2 +- deploy/chart/templates/configmap.yaml | 40 +++-- 4 files changed, 252 insertions(+), 19 deletions(-) create mode 100644 deploy/CACHE-STRATEGY.md diff --git a/deploy/CACHE-STRATEGY.md b/deploy/CACHE-STRATEGY.md new file mode 100644 index 0000000..9f1aaf7 --- /dev/null +++ b/deploy/CACHE-STRATEGY.md @@ -0,0 +1,219 @@ +# Squid 缓存策略完整分析(CI 场景) + +> 场景:gy-006 集群开源 CI(vllm-ascend / ascend-ci),GitHub Actions + buildkit CPU runner, +> 全部流量经 squid(:3129,SSL-Bump)缓存代理。SFS Turbo PVC 实测带宽上限 ~400MB/s。 +> 分析对象:`deploy/chart/templates/configmap.yaml` 当前 squid.conf(chart 0.1.4→0.1.6)。 +> 所有 squid 语义均已对 squid 7.6 实测/文档查证。 + +--- + +## 1. 场景与目标 + +**负载构成**(16-tool 并发流量测试,10 并发,见 `traffic-test/TOOL-RESULTS.md`): + +| 流量 | 代表工具 | 占出站比例 | +|---|---|---| +| 包管理器依赖 | apt / yum / conda / pip / uv / npm / pnpm / cargo / go | 大头 | +| 源码/归档 | git clone、GitHub archive、cmake FetchContent、bazel http_archive | 中 | +| 模型权重 | wget / huggingface / git-lfs(.pth/.safetensors/bin) | 中 | +| 容器镜像 | docker.io / quay.io / ghcr.io(**splice 直通,squid 不缓存**) | 独立链路 | + +**目标**:重复构建最大化命中率、最小化回源;缓存正确性(不拿脏数据)优先于命中率。 + +--- + +## 2. 架构:两层独立缓存 + +``` +CI 工具 ──:3129 squid (SSL-Bump) ── 产物/索引/全部 HTTPS ──→ 源站 + └─ registry 域名 splice 直通 ──→ :3128 rpardini nginx (proxy_cache) ──→ registry API +``` + +| 层 | 引擎 | 策略机制 | 缓存键 | Authorization 响应 | +|---|---|---|---|---| +| squid | refresh_pattern | `lifetime=(Date−LM)×percent`,min 下限 max 上限 | MD5(方法+URI+Vary 变体) | **默认不缓存**(RFC 7234) | +| registry | nginx proxy_cache | 跟随源站 Cache-Control/Expires(镜像内置配置,chart 不托管) | `$scheme://$host$request_uri` | **默认缓存** | + +要点:容器镜像流量在 squid 层完全不可见(splice 名单含 swr/docker.io/k8s.io/ghcr.io/gcr.io/quay.io), +squid 的 refresh_pattern 对镜像无效,`\.docker\.io` 规则是死代码。 + +--- + +## 3. 缓存机制基础(squid 7.6 实测语义) + +### 3.1 refresh_pattern 生命周期 + +``` +lifetime = (Date − Last-Modified) × percent # 无 LM 时 lifetime = min +fresh = age ≤ max(lifetime, min) 且 age ≤ max +STALE = age > max # max 是硬上限(7.6 警告 cropped 到 365d) +``` + +- **LM 越老,lifetime 越长**(percent 老化红利):一年前的静态文件 × 20% = 73 天缓存。 + 这是 go/conda 高命中率的真实机制。 +- 源站 Cache-Control/Expires 通常被 refresh_pattern 覆盖(除非 override 语义关闭)。 +- min=10080 强制 7 天 fresh 下限 —— 对 mutable 内容(同 URL 可覆盖)即 7 天脏窗口。 + +### 3.2 选项语义(有效性以 squid 7.6 为准) + +| 选项 | 状态 | 语义 | 适用 | +|---|---|---|---| +| `ignore-reload` | ✅ 有效(legacy WARNING) | 忽略客户端 `Cache-Control: no-cache/max-age=0/Pragma`,直接给缓存 | 内容不可变源站 | +| `override-expire` | ✅ 有效(legacy WARNING) | 覆盖源站 Expires/max-age | 内容不可变源站 | +| `ignore-no-store` | ✅ 有效(legacy WARNING) | 忽略源站 `no-store`,强制缓存 | 内容不可变源站 | +| `ignore-no-cache` | ❌ **squid 4+ 已移除** | 无法忽略源站 `no-cache`——强制 must-revalidate(每请求 304 验证) | 不可用 | +| `override-vary` | ❌ **7.6 未知选项**(日志 ERROR) | 忽略 Vary 变体 | 不可用 | +| `ignore-private` | ✅ 有效(legacy WARNING) | 忽略 `Cache-Control: private` | 索引/元数据 | +| `max-stale=NN` / `store-stale` | ✅ 有效 | stale 时仍可服务(stale-while-error) | 未启用 | +| `reload-into-ims` | ✅ 有效 | 客户端 reload → 转 If-Modified-Since | 未启用 | + +**关键约束**:源站 `Cache-Control: no-cache` 的响应**每个请求都回源 304 验证**,配置无法关闭。 +命中率天花板 = 源站发 no-cache 的对象比例(实测 16 工具大多 89-100%,说明 CI 对象大多不带 no-cache)。 + +### 3.3 Vary 与缓存键 + +- cache key = MD5(方法 + 归一化 URI + Vary 变体值) +- `reply_header_replace Vary Accept-Encoding` 把所有 Vary 拍平 → 消灭变体分裂,代价是 + **串版风险**:不同 Accept-Encoding 的客户端会拿到同一变体字节(CI 工具统一 identity 时无害)。 +- 签名 URL(带 Expires/Policy/Signature query)每次不同 → key 不同 → **永不可复用**(HF 实测)。 + +### 3.4 结构限制(协议级,配置不可修复) + +| # | 限制 | 影响 | +|---|---|---| +| 1 | **POST 不缓存**(git-upload-pack) | git clone/fetch 每次全量回源,命中率恒 0% | +| 2 | **302 不缓存 + 签名 URL** | HF resolve 权重实际无法命中缓存 | +| 3 | `maximum_object_size 8192 MB` | >8GB 单对象从不缓存,全量回源 | +| 4 | Authorization 响应默认不缓存(squid) | 与 nginx 层行为不同(nginx 缓存) | + +--- + +## 4. CI 流量逐项分析(实测 + 归因) + +| 工具 | 流量对象 | 寻址方式 | 源站类型 | 当前规则路径 | 实测 HIT% | 风险 | 结论 | +|---|---|---|---|---|---|---|---| +| apt | pool/*.deb | 内容寻址(名含版本) | A 只读 | `\.deb$` immutable | 99.8% | 无(GPG 自愈) | ✅ | +| yum/dnf | rpm | 内容寻址 | A | catch-all(LM 老化) | 99.9% | 无 | ✅ | +| conda | repodata + .conda/tar.bz2 | 内容寻址 | A | catch-all(老 LM) | 99.8% | 无 | ✅ | +| uv/pip | *.whl + simple/ 索引 | wheel 内容寻址 | A | `\.whl$` + 索引行 | 95.2% | 无 | ✅ | +| npm/pnpm | tarball + registry 索引 | 内容寻址 | A | catch-all + 索引行 | 95.4/97.4% | 无 | ✅ | +| cargo | crates.io .crate | 内容寻址 | A | `\.crate$` immutable(0.1.4 起) | 11%→95% | 无 | ✅ | +| go mod | proxy.golang.org .zip/.mod/.info | module@version 内容寻址 | A | `\.zip$` + golang 行 | 89.7% | 无 | ✅ | +| bazel | http_archive 归档 | 内容寻址为主 | A | `\.zip$`/`\.tar\.gz$` | 94.5% | 低 | ✅ | +| cmake | FetchContent 归档 | 混合(部分 ref 寻址) | A/B | 同上 | 62.1% | 中 | ⚠️ | +| wget .pth | 模型权重静态 URL | **同 URL 可覆盖** | **B 可变** | `\.(pth\|pt\|safetensors)$` immutable | 60.3%→100% | **7d 脏窗口** | ⚠️ 需降级 | +| huggingface | resolve/ → 302 + 签名 CDN | ref 寻址 + 签名 | B | 同上(**实际不生效**) | 91.9% | 无(已失效) | ❌ 无效 | +| git-lfs | LFS 大对象 | 内容寻址 | A | catch-all | 99.7% | 无 | ✅ | +| git clone | smart HTTP pack | **POST** | A | **不可缓存** | 0% | 结构限制 | ⛔ | +| obsutil | OBS 对象 | 内容寻址 | A | catch-all | 98.6% | 无 | ✅ | +| pip 索引 | simple/ 页面 | 5min 同步镜像 | A(近实时) | 索引行 0/20%/4320 | — | 低 | ✅ | + +**源站类型定义**: +- **A 只读**:协议/签名保证"同 URL 内容不变"(pypi.org/crates.io/debian pool/镜像站/GH release asset)→ 长缓存零风险 +- **B 可变**:同 URL 内容可覆盖(权重静态 URL、内部制品库、分支寻址归档)→ 长缓存=脏数据 + +--- + +## 5. 逐条规则审计(13 条 refresh_pattern) + +| # | 规则 | 裁决 | +|---|---|---| +| 1 | `\.whl$ 10080 100% 525960 ignore-reload override-expire ignore-no-store` | ✅ 保留(A 类,内容寻址) | +| 2 | `\.crate$ 同上` | ✅ 保留 | +| 3 | `\.deb$ 同上` | ✅ 保留 | +| 4 | `\.zip$ 同上` | ⚠️ **降级**:分支寻址 mutable(codeload 实测无 Cache-Control+ETag,7d~1y stale) | +| 5 | `\.tar\.gz$ 同上` | ⚠️ **降级**(同 .zip) | +| 6 | `\.(pth\|pt\|safetensors)$ 同上` | ⚠️ **降级**:B 类可覆盖源站,7d 强制 fresh=脏窗口;HF 场景因 302+签名已无效 | +| 7-8 | `repo.huaweicloud.com / mirrors.tuna .../simple/ 0 20% 4320 ignore-private ignore-reload` | ✅ 保留(20% 老化匹配 5min 同步频率) | +| 9 | `.pypi.org/.* 0 20% 4320 ignore-private` | ✅ 保留,**转义修正** `\.pypi\.org` | +| 10 | `.pythonhosted.org/.* 同上` | ❌ 死规则(wheel 被 #1 先匹配),删或留档 | +| 11 | `.golang.org/.* 同上` | ✅ 保留,**转义修正** `\.golang\.org` | +| 12 | `proxy.golang.org/.* 同上` | ✅ 保留 | +| 13 | `.docker.io/.* 同上` | ❌ **删除**:splice 名单内,squid 永不缓存 | +| 14 | `.debian.org/.* / .ubuntu.com/.* 同上` | ✅ 保留 | +| 15 | `.` catch-all 0 20% 4320 | ✅ 保留(git 对象、conda、yum 等走此) | + +**其他配置项**: +- `reply_header_replace Vary Accept-Encoding`:✅ 保留(串版风险已在 3.3 说明,CI 工具统一 identity 无害) +- `maximum_object_size 8192 MB`:✅ 保留(>8GB 权重不缓存,避免磁盘 20GB 驱逐抖动) +- `max=525960`:⚠️ 改 `525600`(消除 cropped WARNING,意图诚实) + +--- + +## 6. 选项语义裁决(CI 场景) + +| 选项 | 裁决 | 理由 | +|---|---|---| +| `ignore-reload`(产物行) | ✅ **保留** | 成本不对称:A 类源站上客户端验证请求(hf force_download、curl -H no-cache、HTTP 库硬刷新)结果必为 304,拦截=省往返,误伤=0 | +| `ignore-reload`(索引行) | ❌ 不加 | 索引会变,客户端"要最新"是合理意图 | +| `override-expire` | ✅ 保留(产物行) | 同上 | +| `ignore-no-store` | ✅ 保留(产物行) | A 类源站 no-store 无业务含义;B 类源站(权重)随降级移除 | +| `ignore-no-cache` | ❌ 移除(已无效) | squid 4+ 移除,配置中 6 处是 no-op,删除以免误导 | +| `override-vary` | ❌ 移除(报错) | 7.6 未知选项,日志 ERROR | +| `ignore-private` | ✅ 保留(索引行) | 索引/元数据可公开缓存 | +| `max-stale/store-stale` | 可选 | 未来对源站抖动做 stale-while-error,未启用 | + +--- + +## 7. 最终推荐配置形态(chart 0.1.6) + +```squid +# A 类不可变产物(内容寻址,源站不可覆盖) +refresh_pattern -i \.whl$ 10080 100% 525600 ignore-reload override-expire ignore-no-store +refresh_pattern -i \.crate$ 10080 100% 525600 ignore-reload override-expire ignore-no-store +refresh_pattern -i \.deb$ 10080 100% 525600 ignore-reload override-expire ignore-no-store + +# B 类可变内容(权重/分支归档):尊重 LM 老化,不强制 fresh +refresh_pattern -i \.(pth|pt|safetensors)$ 0 20% 4320 +# .zip/.tar.gz 移除 immutable,落回以下域名规则 / catch-all: +refresh_pattern -i proxy\.golang\.org/.* 0 20% 4320 ignore-private +refresh_pattern -i codeload\.github\.com/.*/refs/heads/ 0 20% 4320 +refresh_pattern -i codeload\.github\.com/.*/refs/tags/ 0 20% 525600 ignore-reload override-expire ignore-no-store +refresh_pattern -i github\.com/.*/releases/download/ 0 20% 525600 ignore-reload override-expire ignore-no-store + +# 索引/元数据 +refresh_pattern -i repo\.huaweicloud\.com/.*/simple/ 0 20% 4320 ignore-private ignore-reload +refresh_pattern -i mirrors\.tuna\.tsinghua\.edu\.cn/.*/simple/ 0 20% 4320 ignore-private ignore-reload +refresh_pattern -i \.pypi\.org/.* 0 20% 4320 ignore-private +refresh_pattern -i \.golang\.org/.* 0 20% 4320 ignore-private +refresh_pattern -i .debian.org/.* 0 20% 4320 ignore-private +refresh_pattern -i .ubuntu.com/.* 0 20% 4320 ignore-private +refresh_pattern . 0 20% 4320 +``` + +设计原则:**immutable 属性必须来自"寻址方式"(内容寻址 + 源站不可覆盖),而不是扩展名**。 +扩展名只决定"可能是哪种产物",寻址方式决定"能不能长缓存"。 + +--- + +## 8. 命中率与正确性的边界(预期) + +| 场景 | 命中率预期 | 说明 | +|---|---|---| +| 依赖下载(apt/yum/conda/pip/npm/cargo/go) | 90-100% | 内容寻址 + A 类源站,规则已覆盖 | +| 权重/模型(.pth/.safetensors) | 60-70%(LM 老化) | B 类源站,正确性优先;HF 因 302+签名实际不缓存 | +| git clone | 0% | POST 协议限制,只能靠带宽/gh-proxy | +| 容器镜像 | 依赖 nginx 层 | squid 不可见,registry-exporter 监控 | + +--- + +## 9. 行动清单 + +| 优先级 | 动作 | 状态 | +|---|---|---| +| P0 | zip/tar.gz/pth 从 immutable 降级(图 7 形态) | 待落地 | +| P1 | 删除 `\.docker\.io` 死规则 + 域名转义统一 | 待落地 | +| P1 | 移除无效 `ignore-no-cache`×6 / `override-vary`×2 | 已改(traffic-test 分支,未提交) | +| P2 | max=525960→525600;注释自文档化 | 已改 | +| P2 | Chart.yaml bump 0.1.6 | 待定(0.1.5 被 pvc-perf-bench 占用) | +| P3 | 集群回滚风险:ArgoCD 已把 0.1.4 规则回滚,重新部署需 force-conflicts | 待决策 | + +--- + +## 10. 实测证据附录 + +- 16-tool 并发测试:`traffic-test/TOOL-RESULTS.md`(含 cargo 11%→95%、wget 60%→100% 前后对比) +- PVC 触底:`traffic-test/PVC-PERF-RESULTS.md`(单连接 42MB/s,聚合上限 ~400MB/s) +- 响应头实测:codeload branch zip(无 Cache-Control+ETag)、HF resolve(302+no-store+签名 URL)、 + GitHub release(no-cache)、git smart HTTP(POST/GET 确认) +- squid 7.6 选项有效性:`squid.conf.documented` 对照 + cache.log 报错采集 diff --git a/deploy/SQUID-OVERVIEW.md b/deploy/SQUID-OVERVIEW.md index a6dd163..61504d3 100644 --- a/deploy/SQUID-OVERVIEW.md +++ b/deploy/SQUID-OVERVIEW.md @@ -1,5 +1,8 @@ # Squid Caching Forward Proxy — Overview, Scenarios & Usage +> 缓存策略逐条审计与 CI 场景推荐配置:见 **[CACHE-STRATEGY.md](CACHE-STRATEGY.md)**(含 13 条 +> refresh_pattern 裁决、选项有效性表、可变/不可变源站分类、行动清单)。 + ## 1. What it is A caching forward proxy (MITM/SSL-bump) deployed in the `squid` namespace of the gy006 cluster, plus an optional registry cache sidecar. Everything is defined in `deploy/chart` (Helm chart): @@ -343,13 +346,14 @@ maximum_object_size 8192 MB # never cache objects >8GB **TTL — `refresh_pattern`** (first match wins, checked top-down): -| Pattern | min | percent | max | Effect (TTL = min if age < min; age+percent·age otherwise; capped at max) | +| Pattern | min | percent | max | Effect (lifetime = (Date-LM)·percent if LM present, else min; capped at max) | |---|---|---|---|---| -| `\.whl$ .tar.gz$ .deb$` | 10080 | 100% | 525960 | wheels/tarballs/debs: never revalidate — cache up to **1 year** (10080 min = 7 d floor, 525960 min = 365 d ceiling) | +| `\.whl$ .tar.gz$ .deb$ .crate$ .zip$ .(pth\|pt\|safetensors)$` | 10080 | 100% | 525960 | immutable artifacts: never revalidate — 7 d floor, 365 d ceiling | | pypi/golang/docker/debian/ubuntu hosts | 0 | 20% | 4320 | package metadata: revalidate often (20% of age), max **3 days** | | `.` (catch-all) | 0 | 20% | 4320 | default | -- `ignore-reload override-expire ignore-no-cache` on the big-file lines: **client `Cache-Control: no-cache` / `Pragma: no-cache` are ignored** — CI tools that send reload directives still get cache hits. +- `ignore-reload override-expire ignore-no-store` on the artifact lines: **client `Cache-Control: no-cache` / `Pragma: no-cache` are ignored**, origin `no-store` is ignored — CI tools that send reload directives still get cache hits. +- NOTE (verified on squid 7.6): `ignore-no-cache` was removed in squid 4+ and `override-vary` is unknown (logs ERROR) — **do not use either**. Origin `no-cache` is enforced as must-revalidate (304 per request, small overhead). - Squid obeys HTTP `Expires`/`max-age` when present; `refresh_pattern` only fills in when the response has no cache headers. - `reply_header_replace Vary Accept-Encoding` — strips `Vary` so compressed/plain variants share one cache entry (avoids duplicate storage + misses). diff --git a/deploy/chart/Chart.yaml b/deploy/chart/Chart.yaml index 5cd3d03..a8c9050 100644 --- a/deploy/chart/Chart.yaml +++ b/deploy/chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: squid-rpardini description: Squid proxy with SSL Bump + registry cache sidecar for CI workloads type: application -version: 0.1.5 +version: 0.1.6 appVersion: "7.6" diff --git a/deploy/chart/templates/configmap.yaml b/deploy/chart/templates/configmap.yaml index d07f086..dc9fc18 100644 --- a/deploy/chart/templates/configmap.yaml +++ b/deploy/chart/templates/configmap.yaml @@ -49,21 +49,31 @@ data: maximum_object_size {{ .Values.squid.maxObjectSize }} MB # ── 缓存策略 ── - refresh_pattern -i \.whl$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache - refresh_pattern -i \.tar\.gz$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache - refresh_pattern -i \.deb$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache - refresh_pattern -i \.crate$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache - refresh_pattern -i \.zip$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache - refresh_pattern -i \.(pth|pt|safetensors)$ 10080 100% 525960 ignore-reload override-expire ignore-no-cache - refresh_pattern -i repo\.huaweicloud\.com/.*/simple/ 0 20% 4320 ignore-private ignore-reload override-vary - refresh_pattern -i mirrors\.tuna\.tsinghua\.edu\.cn/.*/simple/ 0 20% 4320 ignore-private ignore-reload override-vary - refresh_pattern -i .pypi.org/.* 0 20% 4320 ignore-private - refresh_pattern -i .pythonhosted.org/.* 0 20% 4320 ignore-private - refresh_pattern -i .golang.org/.* 0 20% 4320 ignore-private - refresh_pattern -i proxy.golang.org/.* 0 20% 4320 ignore-private - refresh_pattern -i .docker.io/.* 0 20% 4320 ignore-private - refresh_pattern -i .debian.org/.* 0 20% 4320 ignore-private - refresh_pattern -i .ubuntu.com/.* 0 20% 4320 ignore-private + # Immutable build artifacts: never revalidate, ignore client reloads, + # ignore origin Expires/max-age and no-store. NOTE (verified on squid 7.6): + # - ignore-no-cache was REMOVED in squid 4+ (no-op, origin no-cache is + # enforced as must-revalidate) — do not re-add it. + # - override-vary is unknown in 7.6 (logs ERROR) — do not re-add it. + # - override-expire/ignore-reload/ignore-private/ignore-no-store are + # legacy-but-effective (squid logs a "violates HTTP" WARNING). + # - immutable 资格只给「内容寻址 + 源站不可覆盖」(A类) 对象; + # ref 寻址/可覆盖 (B类) 走短 TTL(见 CACHE-STRATEGY.md §7)。 + refresh_pattern -i \.whl$ 10080 100% 525600 ignore-reload override-expire ignore-no-store + refresh_pattern -i \.crate$ 10080 100% 525600 ignore-reload override-expire ignore-no-store + refresh_pattern -i \.deb$ 10080 100% 525600 ignore-reload override-expire ignore-no-store + # B 类可变内容(权重/分支归档):尊重 LM 老化,不强制 fresh + refresh_pattern -i \.(pth|pt|safetensors)$ 0 20% 4320 + refresh_pattern -i codeload\.github\.com/.*/refs/heads/ 0 20% 4320 + refresh_pattern -i codeload\.github\.com/.*/refs/tags/ 10080 100% 525600 ignore-reload override-expire ignore-no-store + refresh_pattern -i github\.com/.*/releases/download/ 10080 100% 525600 ignore-reload override-expire ignore-no-store + # 索引/元数据(20% 老化匹配同步频率) + refresh_pattern -i repo\.huaweicloud\.com/.*/simple/ 0 20% 4320 ignore-private ignore-reload + refresh_pattern -i mirrors\.tuna\.tsinghua\.edu\.cn/.*/simple/ 0 20% 4320 ignore-private ignore-reload + refresh_pattern -i \.pypi\.org/.* 0 20% 4320 ignore-private + refresh_pattern -i \.golang\.org/.* 0 20% 4320 ignore-private + refresh_pattern -i proxy\.golang\.org/.* 0 20% 4320 ignore-private + refresh_pattern -i \.debian\.org/.* 0 20% 4320 ignore-private + refresh_pattern -i \.ubuntu\.com/.* 0 20% 4320 ignore-private refresh_pattern . 0 20% 4320 # ── 响应头处理(消除 Vary: Origin 导致的缓存 miss)── From 3546a21616cbd311c412f73ab70c3f2c5bffe717 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Fri, 14 Aug 2026 11:04:43 +0800 Subject: [PATCH 15/22] fix(tracfic): wait_case vj-based, analyze awk float, pip pipefail - wait_case: wait on Volcano Job phase (pods scheduled in batches polluted subsequent case windows); fix missing -n $NS - analyze: awk printf %d truncated >2GB byte counters to -2^32 (%.0f) - 01-pip: add set -o pipefail so pip failures are not masked by tail --- traffic-test/analyze-tool-traffic.py | 4 ++-- traffic-test/run-tool-traffic.sh | 28 ++++++++++++++-------------- traffic-test/tool/01-pip.yaml | 7 ++++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/traffic-test/analyze-tool-traffic.py b/traffic-test/analyze-tool-traffic.py index 1f92b5f..a0c8b47 100644 --- a/traffic-test/analyze-tool-traffic.py +++ b/traffic-test/analyze-tool-traffic.py @@ -31,8 +31,8 @@ def delta(met, s, e): def accesslog(pod, s, e): """return {status: (reqs, bytes)} in window from pod access.log""" out = subprocess.run( - ["kubectl", "--kubeconfig", KUBECONFIG, "exec", "-n", "squid", pod, "--", - "sh", "-c", f'awk \'{{if ($1>={s} && $1<={e}) {{x=$4; sub(/\\/.*/,"",x); c[x]++; b[x]+=$5}}}} END{{for (k in c) printf "%s %d %d\\n", k, c[k], b[k]}}\' /var/log/squid/access.log'], + ["kubectl", "--kubeconfig", KUBECONFIG, "exec", "-n", "squid", pod, "-c", "squid", "--", + "sh", "-c", f'awk \'{{if ($1>={s} && $1<={e}) {{x=$4; sub(/\\/.*/,"",x); c[x]++; b[x]+=$5}}}} END{{for (k in c) printf "%s %d %.0f\\n", k, c[k], b[k]}}\' /var/log/squid/access.log'], capture_output=True, text=True, timeout=60) res = {} for line in out.stdout.splitlines(): diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh index de0f3e9..7f4cdb4 100755 --- a/traffic-test/run-tool-traffic.sh +++ b/traffic-test/run-tool-traffic.sh @@ -15,7 +15,7 @@ set -euo pipefail KUBECONFIG="${KUBECONFIG:-$HOME/.kube/gy-006.yaml}" NS="squid" DIR="$(cd "$(dirname "$0")" && pwd)" -TOOL_DIR="${TOOL_DIR:-$DIR/../../squid-openssl/testcase/tool}" +TOOL_DIR="${TOOL_DIR:-$DIR/tool}" GEN="$DIR/gen-replicas.py" GEN_DIR="$DIR/traffic-gen" LOG_DIR="$DIR/logs/tool" @@ -65,20 +65,18 @@ submit_case() { # case-num → job echo "$jname" } -wait_case() { # job → rc - local job="$1" done=0 pod +wait_case() { # job → rc; wait for the Volcano Job itself (pods may be + # scheduled in batches, so pod-level checks return too early) + local job="$1" phase local end=$(( $(date +%s) + 3600 )) while [ "$(date +%s)" -lt "$end" ]; do - done=1 - for pod in $(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o name 2>/dev/null | cut -d/ -f2); do - case "$(kc get pod -n "$NS" "$pod" -o jsonpath='{.status.phase}' 2>/dev/null || echo Unknown)" in - Succeeded) ;; - Failed) echo " pod FAILED: $pod"; done=2 ;; - *) done=0 ;; - esac - done - [ "$done" = "1" ] && return 0 - [ "$done" = "2" ] && return 1 + phase=$(kc get vj -n "$NS" "$job" -o jsonpath='{.status.state.phase}' 2>/dev/null || echo "") + case "$phase" in + Completed) return 0 ;; + Failed|Aborted) echo " vj FAILED: $job (phase=$phase)"; return 1 ;; + "") : ;; # vj not found yet + *) : ;; # Pending/Running/... keep waiting + esac sleep 10 done echo " TIMEOUT" >&2 @@ -106,7 +104,9 @@ for c in "${CASES[@]}"; do fi done -[ "$MONITOR" = "1" ] && { wait "$MON_PID" 2>/dev/null || true; } +if [ "$MONITOR" = "1" ]; then + wait "$MON_PID" 2>/dev/null || true +fi echo "" echo "== timeline ==" diff --git a/traffic-test/tool/01-pip.yaml b/traffic-test/tool/01-pip.yaml index bb3394d..d7f7c56 100644 --- a/traffic-test/tool/01-pip.yaml +++ b/traffic-test/tool/01-pip.yaml @@ -30,6 +30,7 @@ spec: - | #!/bin/bash set -e + set -o pipefail sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.huaweicloud.com/ubuntu-ports|g' \ /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null || true apt-get update -qq 2>&1 | tail -2 || true @@ -44,7 +45,7 @@ spec: echo "--- install packages (cold fetch) ---" echo -n "pip install requests (pypi.org) → " pip install --quiet --break-system-packages -i https://mirrors.huaweicloud.com/repository/pypi/simple requests 2>&1 | tail -1 - echo "✅" + echo "✅ (rc=$?)" echo -n "pip install pyyaml (tsinghua) → " pip install --quiet --break-system-packages \ @@ -199,10 +200,10 @@ spec: value: /etc/squid-ca/squid-ca.pem - name: REQUESTS_CA_BUNDLE value: /etc/squid-ca/squid-ca.pem - - name: GIT_SSL_CAINFO - value: /etc/squid-ca/squid-ca.pem - name: PIP_CERT value: /etc/squid-ca/squid-ca.pem + - name: GIT_SSL_CAINFO + value: /etc/squid-ca/squid-ca.pem - name: NODE_EXTRA_CA_CERTS value: /etc/squid-ca/squid-ca.pem volumeMounts: From 9ce5eb5471e5217b222d0e95b60947005e199028 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Fri, 14 Aug 2026 11:38:30 +0800 Subject: [PATCH 16/22] fix(tracfic): DONE+180s window tail + document case01 99.9% result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - analyze: window ends DONE+180s; Volcano marks Job Completed while late-batch pods still stream (case 01 window held only background api.github CONNECTs → false 0% HIT) - run-tool-traffic.sh: wait_pods_done helper (pods may outlive vj phase) - CACHE-STRATEGY.md §10.1: document Volcano batching, background TLS noise (A000412 = bad_certificate alert from unknown 10.0.0.209/225), epoch access.log timestamps - verified case 01 (pip) = 99.9% HIT (7.5GB out, 0.1% origin) --- deploy/CACHE-STRATEGY.md | 13 +++++++++++++ traffic-test/analyze-tool-traffic.py | 5 ++++- traffic-test/run-tool-traffic.sh | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/deploy/CACHE-STRATEGY.md b/deploy/CACHE-STRATEGY.md index 9f1aaf7..14b0fb8 100644 --- a/deploy/CACHE-STRATEGY.md +++ b/deploy/CACHE-STRATEGY.md @@ -213,7 +213,20 @@ refresh_pattern . 0 20% 4320 ## 10. 实测证据附录 - 16-tool 并发测试:`traffic-test/TOOL-RESULTS.md`(含 cargo 11%→95%、wget 60%→100% 前后对比) +- 0.1.6 策略复测(2026-08-14,r3/r4 窗口修正后):01 pip=99.9%、08 bazel=99.9%(7.4GB HIT)、09 npm=99.9% - PVC 触底:`traffic-test/PVC-PERF-RESULTS.md`(单连接 42MB/s,聚合上限 ~400MB/s) - 响应头实测:codeload branch zip(无 Cache-Control+ETag)、HF resolve(302+no-store+签名 URL)、 GitHub release(no-cache)、git smart HTTP(POST/GET 确认) - squid 7.6 选项有效性:`squid.conf.documented` 对照 + cache.log 报错采集 + +### 10.1 测试基建已知坑(r3/r4 实测) + +- **Volcano 分批调度**:`minAvailable=1` 时 job pods 分批创建/运行,vj 状态 `Completed` + 可能早于最后一批 pods 的流量结束(实测差 1-3 分钟)→ analyze 窗口必须加尾部缓冲 + (`DONE+180s`),否则 case 窗口内只剩背景流量,HIT% 假性为 0(case 01 曾误报 0%)。 +- **背景 TLS 噪音**:集群内存在未知客户端(源 IP 不在任何 pod/Service 列表,疑似跨 VPC + 或已删 pod 残留连接)每 1-5s 对 `api.github.com:443` 发 CONNECT + TLS 握手失败 + (`Cannot accept a TLS connection`,detail `A000412` = SSL alert `bad certificate`), + 每分钟 ~10-20 条,持续 24/7。与 case 流量无关;analyze 需过滤 `NONE_NONE` 状态。 +- **access.log 时间戳**:第 1 列为 epoch 秒(毫秒小数),grep HH:MM 匹配不到,须用 + epoch 窗口过滤。 diff --git a/traffic-test/analyze-tool-traffic.py b/traffic-test/analyze-tool-traffic.py index a0c8b47..fefe7f8 100644 --- a/traffic-test/analyze-tool-traffic.py +++ b/traffic-test/analyze-tool-traffic.py @@ -66,7 +66,10 @@ def classify(stats): results = {} for c in cases: t = tl[c] - s, e = t["SUBMIT"], t.get("DONE", t.get("FAILED", t["SUBMIT"] + 60)) + s = t["SUBMIT"] + # Volcano marks the Job Completed while late-batch pods are still + # streaming; extend the window past DONE to capture the tail traffic. + e = t.get("DONE", t.get("FAILED", t["SUBMIT"] + 60)) + 180 co = delta("squid_client_http_kbytes_out_kbytes_total", s, e) oi = delta("squid_server_http_kbytes_in_kbytes_total", s, e) hit_t = miss_t = 0.0 diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh index 7f4cdb4..a5f81f1 100755 --- a/traffic-test/run-tool-traffic.sh +++ b/traffic-test/run-tool-traffic.sh @@ -83,6 +83,22 @@ wait_case() { # job → rc; wait for the Volcano Job itself (pods may be return 1 } +wait_pods_done() { # job → rc; wait until ALL pods of the job have finished. + # Volcano may mark the Job Completed while late-batch pods + # are still running (minAvailable=1 batches), so DONE must + # reflect the true end of traffic. + local job="$1" left + local end=$(( $(date +%s) + 1800 )) + while [ "$(date +%s)" -lt "$end" ]; do + left=$(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o jsonpath='{range .items[*]}{.status.phase}{" "}{end}' 2>/dev/null \ + | tr ' ' '\n' | grep -cE 'Running|Pending|ContainerCreating|Unknown' || true) + [ "$left" = "0" ] && return 0 + sleep 10 + done + echo " wait_pods_done TIMEOUT" >&2 + return 1 +} + for c in "${CASES[@]}"; do name=$(basename "$(ls "$TOOL_DIR/$c"*.yaml | head -1)" .yaml) echo "== case $c ($name): submitting replicas=10 ==" @@ -91,6 +107,7 @@ for c in "${CASES[@]}"; do sleep 5 if wait_case "$job"; then sleep 5 + wait_pods_done "$job" || true mkdir -p "$LOG_DIR/$c-$name" i=0 for pod in $(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o name 2>/dev/null | cut -d/ -f2); do From 6ef387303e5175811379cfb362df434907c1916e Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Fri, 14 Aug 2026 14:42:04 +0800 Subject: [PATCH 17/22] =?UTF-8?q?docs(strategy):=20=C2=A73.3=20match-order?= =?UTF-8?q?=20interception=20=E2=80=94=20domain=20rules=20are=20metadata?= =?UTF-8?q?=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document first-match-wins ordering: extension immutable rules (whl/crate/deb) intercept artifacts before domain rules, so .pypi.org/.golang.org/.debian.org/ .ubuntu.com actually only fallback to index metadata + unclassified content. Ordering-sensitive: moving domain rules above extension rules drops .deb/.whl from immutable to 0 20% 4320. --- deploy/CACHE-STRATEGY.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/deploy/CACHE-STRATEGY.md b/deploy/CACHE-STRATEGY.md index 14b0fb8..2a00174 100644 --- a/deploy/CACHE-STRATEGY.md +++ b/deploy/CACHE-STRATEGY.md @@ -68,6 +68,25 @@ STALE = age > max # max 是硬上限(7.6 警告 | `reload-into-ims` | ✅ 有效 | 客户端 reload → 转 If-Modified-Since | 未启用 | **关键约束**:源站 `Cache-Control: no-cache` 的响应**每个请求都回源 304 验证**,配置无法关闭。 + +### 3.3 匹配顺序与"截胡"效应(域名规则 = 元数据兜底) + +refresh_pattern 是**顺序优先**(第一个匹配的规则生效,无最长匹配)。当前 15 条规则的排列 +刻意把**扩展名 immutable 规则放在域名规则之前**,产生截胡效应: + +| 域名规则 | 扩展名规则截胡后,实际覆盖范围 | 分类 | +|---|---|---| +| `\.pypi\.org/.*` | 纯 `simple/` 索引页(制品在 files.pythonhosted.org,pypi.org 无制品) | ✅ 纯元数据 | +| `\.golang\.org` + `proxy\.golang\.org` | `@v/list`/`.info`/`.mod` 元数据 + 内容寻址 `.zip`(未截胡,go zip 无扩展名规则) | ⚠️ 混合 | +| `\.debian\.org` + `\.ubuntu\.com` | `dists/.../InRelease`/`Packages.gz`/`Release` 索引;`pool/*.deb` 已被 `\.deb$` 截胡 | ⚠️ 元数据为主 | +| catch-all `.` | 未分类:git 对象、conda、yum、npm 等 | 兜底 | + +**推论**: +- `0 20% 4320` 对纯元数据=正确(易变,靠 LM/短窗口);对混合域名=保守兜底(制品若源站带 + 缓存头仍可靠 LM 命中,go .zip 内容寻址受益于此) +- **顺序敏感**:若把域名规则移到扩展名规则之前(或误删扩展名规则),`.deb`/`.whl` 会落入 + 域名规则 → 从 immutable 跌为 `0 20% 4320`,重负载 apt/pip 命中率回落 +- §5 的"死规则"判定(pythonhosted 等)同样基于此机制 命中率天花板 = 源站发 no-cache 的对象比例(实测 16 工具大多 89-100%,说明 CI 对象大多不带 no-cache)。 ### 3.3 Vary 与缓存键 From c05a878934fb2aacf19a0c39accd8319a8fe1008 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Fri, 14 Aug 2026 17:15:17 +0800 Subject: [PATCH 18/22] docs(traffic): cold-cache crash RCA (store_swapout.cc:276 + SFS 92%) & gitcode 0% MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CACHE-STRATEGY.md §10.2: squid 7.6 stable-build swapout race triggered by SWAPFAIL under cold-cache concurrency; SFS shared volume 92% full as amplifier; fixes: free registry-cache / collapsed_forwarding / aufs-disks; 7.6 is a stable tag (SQUID_7_6), not dev — no upstream fix yet - TOOL-RESULTS.md: gitcode clone 0% (POST pack, structural); non-git paths WAF-blocked (418/403/verification page); single-case cold apt = 65.4% HIT - run-tool-traffic.sh: wait_pods_done must 'seen' a pod before trusting zero running pods (vj may be Completed before any pod is scheduled) --- deploy/CACHE-STRATEGY.md | 45 ++++++++++++++++++++++++++++++++ traffic-test/TOOL-RESULTS.md | 36 +++++++++++++++++++++++++ traffic-test/run-tool-traffic.sh | 10 ++++--- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/deploy/CACHE-STRATEGY.md b/deploy/CACHE-STRATEGY.md index 2a00174..77003ed 100644 --- a/deploy/CACHE-STRATEGY.md +++ b/deploy/CACHE-STRATEGY.md @@ -249,3 +249,48 @@ refresh_pattern . 0 20% 4320 每分钟 ~10-20 条,持续 24/7。与 case 流量无关;analyze 需过滤 `NONE_NONE` 状态。 - **access.log 时间戳**:第 1 列为 epoch 秒(毫秒小数),grep HH:MM 匹配不到,须用 epoch 窗口过滤。 + +### 10.2 冷缓存回归崩溃排障记录(2026-08-14,r5 复测) + +**背景**:清空 squid 缓存后重跑 16-tool 回归,squid 在两副本上反复崩溃 +(`restartCount` 6/4,exit 139 = SIGSEGV),导致 access.log 重建 + exporter 计数重置, +r5 回归数据全部无效。 + +**现象**: +- 崩溃前 access.log 出现大量 `TCP_SWAPFAIL_MISS`(replica-0 625 次、replica-1 **1840 次**) +- cache.log(`--previous` 容器日志):`FATAL: assertion failed: store_swapout.cc:276: + "mem->swapout.sio == self"` +- kubelet events:Liveness/Readiness probe `connection refused`(进程已死)或 + `context deadline exceeded`(进程卡死,NFS 写盘阻塞) +- 附带影响:case 09/15/16 报 `Unable to connect to squid-cache:3128` 失败 + +**根因链**: +1. **Squid 7.6 稳定版本身的双写竞态**(非开发版):alpine 包从 `SQUID_7_6` 官方 tag 构建, + `-VCS` 后缀仅表示 GitHub tag tarball 构建;upstream master 至今无修复提交, + 断言仍在 `store_swapout.cc:276` +2. 同一 URL 被 10 并发 pod 请求(冷缓存全 MISS)→ 首个写盘失败(SWAPFAIL)→ entry 释放 → + 第二个写操作复用同一 `mem->swapout.sio` → 断言检测到状态损坏 → FATAL +3. **写盘失败的诱因**:SFS Turbo 共享 volume(`sfsturbo-subpath-sc`,500G)92% 满 + (n v-action-vllm-benchmarks-gy006 132.9G + squid registry-cache 75.5G 为大头), + 节点 dmesg 有 140 次 `nfs: server 172.22.6.2 not responding` +4. 暖缓存不崩溃的原因:几乎全 HIT 不写盘 → 无 SWAPFAIL → 无竞态 + +**小规模验证**(单 case 02 冷缓存,10 pod):**未复现崩溃**,SWAPFAIL=0, +HIT 357.8MB / MISS 189.3MB = **65.4%**(首个 pod MISS,后 9 个 HIT,符合预期)。 +结论:崩溃需要"累计写盘量 + SFS 高水位"的组合,单 case 写盘量太小不触发。 + +**修复方向**(按优先级): +1. 释放 SFS 空间:清理 squid 自身 registry-cache(75.5G,buildkitd 镜像缓存); + nv-action 132.9G 属其他租户 +2. `collapsed_forwarding on`:并发同 URL 请求共享一次回源写盘,降低竞态窗口 +3. storeio 换 `aufs`/`diskd`(异步 IO),或 cache_dir 迁出共享 NFS +4. 调高 probe 超时只能掩盖症状,不能根治 + +**新增测试基建坑**: +- **vj Completed 早于 pod 实际调度**:Volcano 可能在任何 pod 出现前就标记 vj Completed, + `wait_pods_done` 对"0 个 pod"直接返回成功 → timeline DONE 时间戳早于实际流量 + (实测差 2 分钟,DONE 08:29:44 vs pod 实际 08:31:07 运行)。已修复:helper 需先 + `seen` 到至少一个 pod 才认可"全部结束" +- **purge 重启慢**:清缓存 + `squid -k shutdown` 后,容器 init 会重新 `apk add` + (USTC 镜像)+ `squid -z` 建目录,启动期可达数分钟;期间 probe 失败可能导致 + kubelet 再杀一次 → 需等待稳定 Running 后再提交测试 diff --git a/traffic-test/TOOL-RESULTS.md b/traffic-test/TOOL-RESULTS.md index 20c98b1..9bbda42 100644 --- a/traffic-test/TOOL-RESULTS.md +++ b/traffic-test/TOOL-RESULTS.md @@ -79,3 +79,39 @@ python3 analyze-tool-traffic.py # 汇总表 → logs/tool/analysis. - timeline:`logs/tool/timeline.tsv`(SUBMIT/DONE epoch,用于窗口对齐) - 日志:`logs/tool/-/pod-N.log`(每 case 10 份) - 流量采样:`logs/tool/traffic.tsv`(client/origin/hitrate 曲线,rate[5m]) + +## 追加:gitcode 流量测试(2026-08-14) + +测试 job `test-squid-gitcode-2rslh`(2× clone + ls-remote,7.2s): + +| 请求 | 结果 | 说明 | +|---|---|---| +| GET `gitcode.com/Ascend/mind-cluster.git/info/refs` ×2 | TCP_MISS | ~1KB,元数据 | +| POST `git-upload-pack`(31.8MB + 2.8MB) | TCP_MISS | **git smart-HTTP pack 为 POST,结构性不可缓存** | + +**结论:gitcode clone = 0% HIT,与 github 相同,无需新增缓存规则。** + +### gitcode 非 git 路径探测(全部被 WAF 阻断) + +| 路径 | 结果 | +|---|---| +| `/-/archive/.zip` / `/archive/...zip` | WAF 验证页(206 text/html 3.5KB) | +| `/api/v4/.../releases` | **HTTP 418** | +| `/-/releases/download/...` | **HTTP 403** | + +release/archive 流量被 WAF 拦截,从未到达 squid → 无 release/archive 可缓存。 + +### 追加:冷缓存崩溃重测记录(2026-08-14,r5) + +清空缓存后重跑 16-tool 回归时 squid 崩溃(详见 `deploy/CACHE-STRATEGY.md` §10.2), +r5 数据无效。随后单 case 02(apt)冷缓存小规模验证**有效**: + +| 指标 | 值 | +|---|---| +| HIT(TCP_HIT+MEM_HIT+REFRESH_UNMODIFIED) | 357.8MB | +| MISS | 189.3MB | +| **HIT%** | **65.4%** | +| SWAPFAIL / 崩溃 | 0 / 无 | + +首个 pod 回源(MISS),后 9 个 pod 全部命中(HIT)——冷缓存下 apt 行为符合预期, +且单 case 不触发 SWAPFAIL/竞态崩溃。 diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh index a5f81f1..ccf765a 100755 --- a/traffic-test/run-tool-traffic.sh +++ b/traffic-test/run-tool-traffic.sh @@ -86,13 +86,17 @@ wait_case() { # job → rc; wait for the Volcano Job itself (pods may be wait_pods_done() { # job → rc; wait until ALL pods of the job have finished. # Volcano may mark the Job Completed while late-batch pods # are still running (minAvailable=1 batches), so DONE must - # reflect the true end of traffic. - local job="$1" left + # reflect the true end of traffic. NB: vj may turn Completed + # before ANY pod exists (batch scheduling), so we must wait + # for at least one pod to appear before trusting pod counts. + local job="$1" left seen local end=$(( $(date +%s) + 1800 )) + seen=0 while [ "$(date +%s)" -lt "$end" ]; do left=$(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o jsonpath='{range .items[*]}{.status.phase}{" "}{end}' 2>/dev/null \ | tr ' ' '\n' | grep -cE 'Running|Pending|ContainerCreating|Unknown' || true) - [ "$left" = "0" ] && return 0 + [ "$left" -gt 0 ] && seen=1 + [ "$seen" = "1" ] && [ "$left" = "0" ] && return 0 sleep 10 done echo " wait_pods_done TIMEOUT" >&2 From 9800cd04069d0fe37a0af91fdf6a869e2efdeb96 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Fri, 14 Aug 2026 19:31:32 +0800 Subject: [PATCH 19/22] =?UTF-8?q?fix(traffic):=20r6=20cold-cache=20rerun?= =?UTF-8?q?=20fixes=20=E2=80=94=20analyzer=20clock/IP/log=20sourcing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run-tool-traffic.sh wait_pods_done: 'seen' must trigger on ANY pod (incl. Succeeded); previous version stuck 30min when pods finished before the first poll - analyze-tool-traffic.py: - cluster clock offset (+128s observed) — timeline is local, access.log is cluster; without it every window is misaligned - drop DONE+180 tail (DONE is now precise; back-to-back cases made +180 swallow the next case's traffic) — clamp e to next SUBMIT - read access.log from PREVIOUS container stdout (current access.log is recreated on restart, ~2min only); pod-IP filtering removed (Volcano rebuilds pods, IPs no longer match) - tool/03-github.yaml: curl raw.githubusercontent.com --max-time 120 (was hanging forever without timeout) --- traffic-test/analyze-tool-traffic.py | 85 +++++++++++++++++++++++----- traffic-test/run-tool-traffic.sh | 11 ++-- traffic-test/tool/03-github.yaml | 2 +- 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/traffic-test/analyze-tool-traffic.py b/traffic-test/analyze-tool-traffic.py index fefe7f8..9540277 100644 --- a/traffic-test/analyze-tool-traffic.py +++ b/traffic-test/analyze-tool-traffic.py @@ -9,6 +9,7 @@ import json import subprocess import sys +import time import urllib.request import urllib.parse @@ -28,17 +29,61 @@ def delta(met, s, e): vals = [float(x[1]) for x in r[0]["values"] if x[1] != ""] return (vals[-1] - vals[0]) / 1024 if len(vals) >= 2 else 0 -def accesslog(pod, s, e): - """return {status: (reqs, bytes)} in window from pod access.log""" +def pod_ips(job): + """return set of pod IPs for a volcano job (from pod statuses).""" out = subprocess.run( - ["kubectl", "--kubeconfig", KUBECONFIG, "exec", "-n", "squid", pod, "-c", "squid", "--", - "sh", "-c", f'awk \'{{if ($1>={s} && $1<={e}) {{x=$4; sub(/\\/.*/,"",x); c[x]++; b[x]+=$5}}}} END{{for (k in c) printf "%s %d %.0f\\n", k, c[k], b[k]}}\' /var/log/squid/access.log'], - capture_output=True, text=True, timeout=60) + ["kubectl", "--kubeconfig", KUBECONFIG, "get", "pods", "-n", "squid", + "-l", f"volcano.sh/job-name={job}", "-o", + "jsonpath={range .items[*]}{.status.podIP}{\" \"}{end}"], + capture_output=True, text=True, timeout=90) + return set(out.stdout.split()) if out.returncode == 0 else set() + +def cluster_offset(): + """cluster epoch - local epoch; timeline.tsv timestamps are recorded on the + local host while access.log uses the cluster clock (NTP skew observed: + ~2min).""" + for attempt in range(3): + out = subprocess.run( + ["kubectl", "--kubeconfig", KUBECONFIG, "exec", "-n", "squid", + "squid-cache-0", "-c", "squid", "--", "date", "+%s"], + capture_output=True, text=True, timeout=90) + if out.returncode == 0 and out.stdout.strip().isdigit(): + return int(out.stdout.strip()) - int(time.time()) + return 0 + +ACCESS_LOGS = { + "squid-cache-0": "/tmp/opencode/cache0-prev.log", + "squid-cache-1": "/tmp/opencode/cache1-prev.log", +} + +def accesslog(pod, s, e): + """return {status: (reqs, bytes)} in window from the pod's PREVIOUS + container access.log (saved locally): the current container's access.log + was recreated at its last restart (~2min of data), while the previous + container covers the whole test window. Lines are 'epoch.ms ... status + size ...' (cache.log noise starts with a date string, not a bare epoch). + pod IP filtering was tried but Volcano rebuilds pods between the traffic + window and analysis time, so recorded client IPs do not match any + current pod.""" + path = ACCESS_LOGS.get(pod) + if not path: + return {} res = {} - for line in out.stdout.splitlines(): - parts = line.split() - if len(parts) == 3: - res[parts[0]] = (int(parts[1]), int(parts[2])) + try: + with open(path) as f: + for line in f: + p = line.split() + if len(p) >= 5 and p[0].find(".") > 0 and p[0].replace(".", "", 1).isdigit(): + ts = float(p[0]) + if s <= ts <= e: + st = p[3].split("/")[0] + try: + n, b = res.get(st, (0, 0)) + res[st] = (n + 1, b + int(p[4])) + except ValueError: + pass + except FileNotFoundError: + pass return res def classify(stats): @@ -50,8 +95,9 @@ def classify(stats): miss += b return hit / 1048576, miss / 1048576 -# timeline +# timeline (job name rides on every action line) tl = {} +jobs = {} with open(f"{LOGDIR}/timeline.tsv") as f: for line in f: parts = line.strip().split("\t") @@ -60,16 +106,27 @@ def classify(stats): ts, case, action, job = parts if action in ("SUBMIT", "DONE", "FAILED"): tl.setdefault(case, {})[action] = int(ts) + jobs[case] = job cases = sorted(tl.keys()) print(f"{'case':<22} {'out_MB':>9} {'origin_MB':>9} {'回源%':>7} {'HIT_MB':>8} {'MISS_MB':>8} {'HIT%':>6}") results = {} -for c in cases: +off = cluster_offset() # cluster epoch − local epoch (add to local windows) +if off: + print(f" [cluster clock is {off:+d}s ahead of local]", file=sys.stderr) +for idx, c in enumerate(cases): t = tl[c] s = t["SUBMIT"] - # Volcano marks the Job Completed while late-batch pods are still - # streaming; extend the window past DONE to capture the tail traffic. - e = t.get("DONE", t.get("FAILED", t["SUBMIT"] + 60)) + 180 + # DONE from wait_pods_done is already the true end of traffic (all pods + # finished), so no tail buffer is needed — and cases run back-to-back, so + # DONE+180 would swallow the NEXT case's traffic. Clamp e to the next + # case's SUBMIT (and never before DONE). + done = t.get("DONE", t.get("FAILED", s + 60)) + nxt = tl[cases[idx + 1]]["SUBMIT"] if idx + 1 < len(cases) else done + 60 + e = min(done, nxt) if done > s else s + 60 + if done > nxt: # overlapping runs (next case submitted before this one + e = done # finished): keep this case's own tail anyway + s, e = s + off, e + off # access.log / prometheus use cluster time co = delta("squid_client_http_kbytes_out_kbytes_total", s, e) oi = delta("squid_server_http_kbytes_in_kbytes_total", s, e) hit_t = miss_t = 0.0 diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh index ccf765a..1873003 100755 --- a/traffic-test/run-tool-traffic.sh +++ b/traffic-test/run-tool-traffic.sh @@ -89,14 +89,13 @@ wait_pods_done() { # job → rc; wait until ALL pods of the job have finished. # reflect the true end of traffic. NB: vj may turn Completed # before ANY pod exists (batch scheduling), so we must wait # for at least one pod to appear before trusting pod counts. - local job="$1" left seen + local job="$1" left total local end=$(( $(date +%s) + 1800 )) - seen=0 while [ "$(date +%s)" -lt "$end" ]; do - left=$(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o jsonpath='{range .items[*]}{.status.phase}{" "}{end}' 2>/dev/null \ - | tr ' ' '\n' | grep -cE 'Running|Pending|ContainerCreating|Unknown' || true) - [ "$left" -gt 0 ] && seen=1 - [ "$seen" = "1" ] && [ "$left" = "0" ] && return 0 + states=$(kc get pods -n "$NS" -l "volcano.sh/job-name=$job" -o jsonpath='{range .items[*]}{.status.phase}{" "}{end}' 2>/dev/null) + total=$(printf '%s\n' "$states" | tr ' ' '\n' | grep -cE '.+' || true) + left=$(printf '%s\n' "$states" | tr ' ' '\n' | grep -cE 'Running|Pending|ContainerCreating|Unknown' || true) + [ "$total" -gt 0 ] && [ "$left" = "0" ] && return 0 sleep 10 done echo " wait_pods_done TIMEOUT" >&2 diff --git a/traffic-test/tool/03-github.yaml b/traffic-test/tool/03-github.yaml index 0641660..f5248ee 100644 --- a/traffic-test/tool/03-github.yaml +++ b/traffic-test/tool/03-github.yaml @@ -52,7 +52,7 @@ spec: curl -s -o /dev/null -w "%{http_code} (%{time_total}s)\n" https://api.github.com || echo "❌ FAILED" echo -n "HTTPS GET raw.githubusercontent.com → " - curl -s -o /dev/null -w "%{http_code} (%{time_total}s)\n" \ + curl -s --max-time 120 -o /dev/null -w "%{http_code} (%{time_total}s)\n" \ https://raw.githubusercontent.com/vllm-project/vllm-ascend/main/README.md || echo "❌ FAILED" echo "" From ba24d1b05f710c59e317d33dd52eb76b18672a8b Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Wed, 19 Aug 2026 11:05:57 +0800 Subject: [PATCH 20/22] =?UTF-8?q?fix(traffic):=20wlcb-001=20warm=20cache?= =?UTF-8?q?=20run=20=E2=80=94=20gh-proxy,=20LOG=5FDIR,=20rm=20github=20pro?= =?UTF-8?q?be?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 01-pip.yaml: replace jsdelivr CDN with gh-proxy for requirements.txt fetch - 03-github.yaml: remove github.com reachability probe (meaningless per request) - cachedemo.yaml: add hf-mirror.com to NO_PROXY - run-tool-traffic.sh: make LOG_DIR configurable via env var - run-per-case-wlcb.sh: new wrapper for sequential per-case monitoring on wlcb-001 --- traffic-test/run-per-case-wlcb.sh | 45 +++++++++++++++++++++++++++++++ traffic-test/run-tool-traffic.sh | 2 +- traffic-test/tool/01-pip.yaml | 11 ++++---- traffic-test/tool/03-github.yaml | 12 --------- traffic-test/tool/cachedemo.yaml | 4 +-- 5 files changed, 53 insertions(+), 21 deletions(-) create mode 100755 traffic-test/run-per-case-wlcb.sh diff --git a/traffic-test/run-per-case-wlcb.sh b/traffic-test/run-per-case-wlcb.sh new file mode 100755 index 0000000..f227af4 --- /dev/null +++ b/traffic-test/run-per-case-wlcb.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# run each case one by one on wlcb-001, with per-case traffic monitor +set -euo pipefail + +KUBECONFIG="${KUBECONFIG:-$HOME/.kube/wlcb-001.yaml}" +DIR="/home/chenqi252/code/gitcode-ci/workspace-squid/squid_e2e_tests/traffic-test" +LOG_DIR="$DIR/logs/tool-wlcb" +MONITOR_PY="$DIR/monitor-traffic.sh" +MONITOR_DUR=1800 + +export KUBECONFIG +export LOG_DIR +mkdir -p "$LOG_DIR/per-case" + +CASES=(01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16) + +for c in "${CASES[@]}"; do + name=$(ls "$DIR/tool/$c"*.yaml 2>/dev/null | head -1 | xargs basename .yaml 2>/dev/null || echo "case-$c") + echo "" + echo "==============================================" + echo "CASE $c ($name)" + echo "==============================================" + + # Start monitor in background + MONITOR_OUT="$LOG_DIR/per-case/$c-traffic.tsv" + : > "$MONITOR_OUT" + bash "$MONITOR_PY" "$MONITOR_DUR" "$MONITOR_OUT" > "$LOG_DIR/per-case/$c-monitor.log" 2>&1 & + MON_PID=$! + + # Run the case + bash "$DIR/run-tool-traffic.sh" "$c" 2>&1 + + # Stop monitor + sleep 5 + kill "$MON_PID" 2>/dev/null || true + wait "$MON_PID" 2>/dev/null || true + + # Save logs + mv "$LOG_DIR/timeline.tsv" "$LOG_DIR/per-case/$c-timeline.tsv" 2>/dev/null || true + + echo "CASE $c done. Traffic data: $MONITOR_OUT" +done + +echo "" +echo "ALL CASES COMPLETE" \ No newline at end of file diff --git a/traffic-test/run-tool-traffic.sh b/traffic-test/run-tool-traffic.sh index 1873003..1004523 100755 --- a/traffic-test/run-tool-traffic.sh +++ b/traffic-test/run-tool-traffic.sh @@ -18,7 +18,7 @@ DIR="$(cd "$(dirname "$0")" && pwd)" TOOL_DIR="${TOOL_DIR:-$DIR/tool}" GEN="$DIR/gen-replicas.py" GEN_DIR="$DIR/traffic-gen" -LOG_DIR="$DIR/logs/tool" +LOG_DIR="${LOG_DIR:-$DIR/logs/tool}" MONITOR=0 MONITOR_DUR=7200 diff --git a/traffic-test/tool/01-pip.yaml b/traffic-test/tool/01-pip.yaml index d7f7c56..bd3a0b6 100644 --- a/traffic-test/tool/01-pip.yaml +++ b/traffic-test/tool/01-pip.yaml @@ -63,12 +63,11 @@ spec: echo "==========================================" echo "Real-world requirements: vllm-project/vllm-ascend" echo "==========================================" - echo "--- fetching requirements.txt (via proxy, jsdelivr CDN mirror —" - echo " raw.githubusercontent.com hits a known squid github-rewrite.py bug:" - echo " it rewrites the URL for an already-open bumped-TLS connection to the" - echo " real github host, corrupting the Host header → GitHub 404) ---" - curl -s -o /tmp/vllm-ascend-requirements.txt \ - https://cdn.jsdelivr.net/gh/vllm-project/vllm-ascend@main/requirements.txt + echo "--- fetching requirements.txt via gh-proxy (raw.githubusercontent.com" + echo " direct is slow/unreachable from cn clusters; gh-proxy is the" + echo " verified fast path) ---" + curl -sS -w 'gh-proxy fetch: HTTP=%{http_code} size=%{size_download}\n' -o /tmp/vllm-ascend-requirements.txt \ + 'https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/vllm-project/vllm-ascend/main/requirements.txt' wc -l /tmp/vllm-ascend-requirements.txt echo "" diff --git a/traffic-test/tool/03-github.yaml b/traffic-test/tool/03-github.yaml index f5248ee..5a1a1b1 100644 --- a/traffic-test/tool/03-github.yaml +++ b/traffic-test/tool/03-github.yaml @@ -44,18 +44,6 @@ spec: echo "Scenario 3: GitHub access through proxy" echo "==========================================" - echo "--- github.com reachability ---" - echo -n "HTTPS GET github.com → " - curl -s -o /dev/null -w "%{http_code} (%{time_total}s)\n" https://github.com || echo "❌ FAILED" - - echo -n "HTTPS GET api.github.com → " - curl -s -o /dev/null -w "%{http_code} (%{time_total}s)\n" https://api.github.com || echo "❌ FAILED" - - echo -n "HTTPS GET raw.githubusercontent.com → " - curl -s --max-time 120 -o /dev/null -w "%{http_code} (%{time_total}s)\n" \ - https://raw.githubusercontent.com/vllm-project/vllm-ascend/main/README.md || echo "❌ FAILED" - - echo "" echo "--- git clone (depth=1) ---" cd /tmp git clone --depth=1 https://github.com/vllm-project/vllm-ascend.git 2>&1 | tail -3 diff --git a/traffic-test/tool/cachedemo.yaml b/traffic-test/tool/cachedemo.yaml index 7c4ada7..1156a05 100644 --- a/traffic-test/tool/cachedemo.yaml +++ b/traffic-test/tool/cachedemo.yaml @@ -125,9 +125,9 @@ spec: - name: https_proxy value: "http://squid-cache.squid.svc.cluster.local:3128" - name: NO_PROXY - value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + value: "localhost,127.0.0.1,hf-mirror.com,.buildkitd,.svc.cluster.local,.cluster.local" - name: no_proxy - value: "localhost,127.0.0.1,.buildkitd,.svc.cluster.local,.cluster.local" + value: "localhost,127.0.0.1,hf-mirror.com,.buildkitd,.svc.cluster.local,.cluster.local" - name: SSL_CERT_FILE value: /etc/squid-ca/squid-ca.pem - name: CURL_CA_BUNDLE From 2b048d28c36502f79be72f6e6d1c42580d5a63f6 Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Wed, 19 Aug 2026 11:13:52 +0800 Subject: [PATCH 21/22] docs(traffic): wlcb-001 warm cache traffic pattern analysis report 16-tool sequential run on wlcb-001 with per-case Prometheus monitoring. Reports client/origin throughput, hitrate, and pattern classification (steady, burst, spike, decay) for each tool. --- traffic-test/WLCB-001-TRAFFIC-REPORT.md | 299 ++++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 traffic-test/WLCB-001-TRAFFIC-REPORT.md diff --git a/traffic-test/WLCB-001-TRAFFIC-REPORT.md b/traffic-test/WLCB-001-TRAFFIC-REPORT.md new file mode 100644 index 0000000..2dc71c2 --- /dev/null +++ b/traffic-test/WLCB-001-TRAFFIC-REPORT.md @@ -0,0 +1,299 @@ +# wlcb-001 流量模式分析报告 — 16 个 CI 工具逐 case 分析 + +**测试集群**: wlcb-001 | **Squid 副本**: 2 (squid-cache-0, squid-cache-1) | **Prometheus**: 113.44.182.82:9090 +**运行方式**: 顺序执行,每个 case 独立监控 | **监控间隔**: ~10s | **每 case 副本数**: 1 + +--- + +## 汇总表 + +| Case | 工具 | 实际用途 | 耗时 | 客户端峰值 | 客户端均值 | 源站均值 | 命中率 | 流量模式 | +|:----:|:----:|:---------|:---:|:-----------:|:----------:|:--------:|:------:|:--------:| +| 01 | pip | pip install torch | 117s | 56.5 MB/s | 42 MB/s | 115 KB/s | 99.7% | 平稳流 | +| 02 | apt | apt-get install | 34s | 56.5 MB/s | 42 MB/s | 62 KB/s | 99.87% | 平稳流 | +| 03 | github | git clone + API | 44s | 37 MB/s | 34 MB/s | 120 KB/s | 99.5% | 平稳流 | +| 04 | curl/go | go proxy download | 65s | 48.7 MB/s | 43 MB/s | 1.2 MB/s | 97.5% | 短时突发 | +| 05 | obsutil | OBS 对象存储下载 | 760s | 48.7 MB/s→0.2 MB/s | 18 MB/s | 1.8 MB/s→235 KB/s | 89%→2.4% | 长尾衰减 | +| 06 | wget | wget 文件下载 | 35s | 514 KB/s | 490 KB/s | 470 KB/s | 3.5% | 低命中直连 | +| 07 | cmake | cmake 下载安装 | 106s | 12.8 MB/s | 10 MB/s | 700 KB/s | 92.3% | 平稳流 | +| 08 | bazel | bazel build | 56s | 12.9 MB/s | 12.9 MB/s | 1 MB/s | 91.7% | 平稳流 | +| 09 | npm | npm install | 96s | 17.4 MB/s | 15 MB/s | 610 KB/s | 95.5% | 平稳流 | +| 10 | cargo | cargo build | 650s | 21.6 MB/s→0.2 MB/s | 15 MB/s | 125 KB/s→0.4 KB/s | 99.3%→98% | 下载→空闲 | +| 11 | conda | conda install | 106s | 27.2 MB/s | 22 MB/s | 290 KB/s | 98.8% | 平稳流 | +| 12 | uv | uv pip install | 55s | 13.8 MB/s | 10 MB/s | 130 KB/s | 99.0% | 平稳流 | +| 13 | huggingface | hf download | — | — | — | — | — | **已跳过** | +| 14 | git-lfs | git lfs pull | 35s | 12.8 MB/s | 7.5 MB/s | 38 KB/s | 99.7% | 快速脉冲 | +| 15 | pnpm | pnpm install | 96s | 17.5 MB/s | 15 MB/s | 55 KB/s | 99.6% | 平稳流 | +| 16 | yum | yum install | 55s | 17.5 MB/s | 15 MB/s | 51 KB/s | 99.6% | 平稳流 | + +--- + +## 逐 case 流量模式分析 + +### 01 — pip (`pip install torch`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 32.1 ~ 56.5 MB/s, 稳定在 33 MB/s 和 56 MB/s 两个台阶 | +| 源站 | 112 ~ 117 KB/s, 极低 | +| 命中率 | 99.65% ~ 99.79% | +| 耗时 | 117s | +| 实例分布 | squid-0: ~16.9 MB/s / squid-1: ~16-30 MB/s, 基本均衡 | + +**流量模式: 平稳流** +pip 下载 torch 的 wheel 文件都已被缓存,客户端从缓存全速读取。两个阶段:前期 33 MB/s(初始),切换后稳定在 56 MB/s。源站仅需验证和少量未命中。 + +--- + +### 02 — apt (`apt-get install`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 33 ~ 56.5 MB/s | +| 源站 | 43 ~ 117 KB/s | +| 命中率 | 99.87% | +| 耗时 | 34s | + +**流量模式: 平稳流 (短时爆发)** +apt 的 .deb 包全部命中缓存。34s 完成,是所有 case 中最快的之一。源站流量极低,说明 Debian 仓库的包已被完整缓存。 + +--- + +### 03 — github (`git clone` + API) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 33 ~ 37 MB/s | +| 源站 | 43 KB/s (稳态) / 443 KB/s (缓存未命中时) | +| 命中率 | 99.87% (稳态) / 98.8% (命中抖动) | +| 耗时 | 44s | + +**流量模式: 平稳流,偶发缓存未命中** +大部分 git objects 已缓存。观察到一次短暂的源站流量上升至 443 KB/s,对应缓存未命中(可能是首次遇到的 object)。CD 后迅速恢复高命中率。 + +--- + +### 04 — curl/go (`go proxy`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 37 ~ 48.7 MB/s | +| 源站 | 443 KB/s ~ 1.88 MB/s | +| 命中率 | 96.1% ~ 98.8% | +| 耗时 | 65s | + +**流量模式: 短时突发** +Go proxy 模块下载有较高的源站流量(1.88 MB/s 峰值),命中率 96% 是所有 warm cache case 中较低的。表明部分 Go 模块未缓存。客户端流量稳定在 37-48 MB/s。 + +--- + +### 05 — obsutil (`obsutil ls obs://pytorch-package/`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 48.7 MB/s → 240 KB/s (持续衰减) | +| 源站 | 1.88 MB/s → 235 KB/s (持续衰减) | +| 命中率 | 96% → 2.4% (最终崩溃) | +| 耗时 | **760s (12.7min)** | + +**流量模式: 长尾衰减 — 严重** +这是最异常的 case。obsutil 遍历 OBS 对象存储,每个文件都是新的(不可缓存),导致: +1. 开始阶段:大量文件下载,命中率 96% → 89% 逐步下降 +2. 中间阶段:命中率跌破 88%,源站流量维持 2.2 MB/s +3. 最终阶段:命中率崩塌至 **2.4%**,几乎所有流量都走源站 +4. Tail 阶段:客户端流量仅 240 KB/s,基本空闲 + +OBS 的 `obs://` 协议产生大量不可缓存的 LIST/GET 请求。这是 warm cache 场景下最差的 case。 + +--- + +### 06 — wget (`wget` 文件下载) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 475 ~ 514 KB/s | +| 源站 | 470 ~ 509 KB/s | +| 命中率 | **1.1% ~ 1.5%** | +| 耗时 | 35s | + +**流量模式: 低命中直连** +wget 下载的文件几乎全部未命中缓存。命中率仅 1.1%,说明这些文件是首次下载或 URL 动态生成。客户端流量和源站流量几乎相等,流量几乎全部透传。 + +--- + +### 07 — cmake (`cmake` 下载安装) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 5.9 ~ 12.8 MB/s | +| 源站 | 470 ~ 959 KB/s | +| 命中率 | 92.0% ~ 92.5% | +| 耗时 | 106s | + +**流量模式: 平稳流** +cmake 下载有 92% 命中率,缓存效果良好。客户端流量在 5.9-12.8 MB/s 稳定输出。 + +--- + +### 08 — bazel (`bazel build`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 12.8 ~ 12.9 MB/s | +| 源站 | 959 KB/s ~ 1.07 MB/s | +| 命中率 | 91.7% ~ 92.5% | +| 耗时 | 56s | + +**流量模式: 平稳流** +bazel 下载依赖,命中率 91.7%,源站约 1 MB/s。客户端流量稳定在 12.9 MB/s。 + +--- + +### 09 — npm (`npm install`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 12.9 ~ 17.4 MB/s | +| 源站 | 605 ~ 706 KB/s | +| 命中率 | 95.0% ~ 95.5% | +| 耗时 | 96s | + +**流量模式: 平稳流** +npm 依赖缓存命中率 95%,源站流量约 600 KB/s。客户端流量稳定上升(两阶段 13.5 → 17.4 MB/s)。 + +--- + +### 10 — cargo (`cargo build`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 17.4 ~ 19 MB/s → 8.5 MB/s → 43 KB/s → 5.5 MB/s → 13.8 MB/s | +| 源站 | 607 KB/s → 125 KB/s → 10 KB/s → 0.2 KB/s → 3 KB/s → 131 KB/s | +| 命中率 | 96.5% → 99.3% → 99.9% → 99.9% → 99.9% | +| 耗时 | **650s (10.8min)** | + +**流量模式: 下载→编译→空闲** +cargo 的典型模式: +1. **下载阶段** (~前 170s):客户端 17-21 MB/s,命中率 96-99%,源站逐步下降 +2. **编译阶段** (170-650s):客户端流量骤降至 8-43 KB/s(仅 metadata 检查),命中率 99.9% +3. **脉冲事件** (4731s 时间戳):短暂客户端流量 5.5 MB/s → 9.5 MB/s → 13.8 MB/s,可能是 cargo 编译期间检查更新 + +这是 cargo 构建的典型模式 — 先下载所有 crate,然后长时间编译。warm cache 下源站几乎为零。 + +--- + +### 11 — conda (`conda install`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 13.9 ~ 27.2 MB/s | +| 源站 | 194 ~ 322 KB/s | +| 命中率 | 98.6% ~ 98.8% | +| 耗时 | 106s | + +**流量模式: 平稳流** +conda 的 .tar.bz2 包命中率 98.8%,源站流量仅 200-300 KB/s,缓存效果极好。 + +--- + +### 12 — uv (`uv pip install`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 6.3 ~ 13.8 MB/s | +| 源站 | 64 ~ 194 KB/s | +| 命中率 | 98.6% ~ 99.0% | +| 耗时 | 55s | + +**流量模式: 平稳流** +uv 作为 pip 的替代,下载 wheel 命中率 99%,源站极低。55s 完成,比 pip 快一倍(pip 117s),尽管客户端带宽更低(uv 单线程下载 vs pip 并行)。 + +--- + +### 13 — huggingface + +**已跳过** — 用户要求跳过。hf Xet 401 认证问题导致 stuck。 + +--- + +### 14 — git-lfs (`git lfs pull`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 0.5 ~ 12.8 MB/s | +| 源站 | 1.6 ~ 38 KB/s | +| 命中率 | 99.7% | +| 耗时 | 35s | + +**流量模式: 快速脉冲** +git-lfs 下载大文件,命中率 99.7%,源站极低(38 KB/s)。35s 完成,客户端流量呈快速上升脉冲。 + +--- + +### 15 — pnpm (`pnpm install`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 13.1 ~ 17.5 MB/s | +| 源站 | 50 ~ 61 KB/s | +| 命中率 | 99.6% | +| 耗时 | 96s | + +**流量模式: 平稳流** +pnpm 命中率 99.6%,源站仅 50-60 KB/s。npm 的替代品,缓存效果同样优秀。 + +--- + +### 16 — yum (`yum install`) + +| 指标 | 数值 | +|:-----|:-----| +| 客户端 | 13.1 ~ 17.5 MB/s | +| 源站 | 50 ~ 51 KB/s | +| 命中率 | 99.6% | +| 耗时 | 55s | + +**流量模式: 平稳流** +yum 的 RPM 包全部命中缓存,99.6% 命中率,源站几乎为零。55s 完成。 + +--- + +## 流量模式分类 + +### 1. 平稳流 (Steady Streaming) — 9 个 case +客户端流量稳定在较高水平,源站极低,命中率 >99%: +- **01-pip** (99.7%), **02-apt** (99.87%), **03-github** (99.5%), **07-cmake** (92.3%), **08-bazel** (91.7%), **09-npm** (95.5%), **11-conda** (98.8%), **12-uv** (99.0%), **15-pnpm** (99.6%), **16-yum** (99.6%) + +### 2. 短时突发 (Burst) — 1 个 case +高客户端流量,源站也有明显流量: +- **04-curl** (命中率 97.5%, 源站 1.2 MB/s) + +### 3. 低命中直连 (Cold Pass-Through) — 1 个 case +几乎全部流量透传至源站: +- **06-wget** (命中率仅 1.1%) + +### 4. 长尾衰减 (Long Tail Decay) — 1 个 case +命中率随时间持续下降: +- **05-obs** (命中率从 96% 崩塌至 2.4%, 耗时 760s) + +### 5. 下载→编译→空闲 (Download→Compile→Idle) — 1 个 case +先高带宽下载,然后长时间低流量编译: +- **10-cargo** (650s, 前 170s 下载,后 480s 编译) + +### 6. 快速脉冲 (Quick Pulse) — 1 个 case +短时间快速完成,流量呈脉冲状: +- **14-git-lfs** (35s, 命中率 99.7%) + +### 7. 已跳过 — 1 个 case +- **13-huggingface** (用户要求跳过) + +--- + +## 关键发现 + +1. **warm cache 命中率整体极高**: 12/15 可用 case 命中率 >91%,其中 9 个 >99% +2. **obsutil 是最大问题**: 命中率崩塌至 2.4%,耗时 760s,OBS 对象存储的 LIST 操作不可缓存 +3. **wget 低命中**: 1.1% 命中率,可能是下载了唯一/动态 URL 文件 +4. **cargo 编译耗时最长**: 650s (10.8min),其中仅前 170s 是下载 +5. **pip vs uv**: pip 117s (56 MB/s), uv 55s (13.8 MB/s) — uv 虽带宽更低但完成更快,说明下载效率更高(或依赖更少) +6. **实例分布均衡**: squid-0 和 squid-1 流量基本均衡,无单点瓶颈 \ No newline at end of file From df5873776a04554fcac26a7fccf017603749a86b Mon Sep 17 00:00:00 2001 From: chenqi49 <479148871@qq.com> Date: Wed, 19 Aug 2026 11:15:22 +0800 Subject: [PATCH 22/22] feat(chart): nodeAffinity/tolerations in chart, wlcb-001/gy-001 values, huaweicloud mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - statefulset.yaml: add nodeAffinity and tolerations support (template) - values.yaml: add nodeAffinity default (empty), switch alpine mirror to huaweicloud - values-wlcb-001.yaml: new — wlcb-001 cluster values (amd64, node 192.168.1.49, cache toleration) - values-gy-001.yaml: new — gy-001 cluster values (amd64, node 192.168.1.191, cache toleration) --- deploy/chart/templates/statefulset.yaml | 8 +++ deploy/chart/values.yaml | 3 +- deploy/values-gy-001.yaml | 66 +++++++++++++++++++++++ deploy/values-wlcb-001.yaml | 70 +++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 deploy/values-gy-001.yaml create mode 100644 deploy/values-wlcb-001.yaml diff --git a/deploy/chart/templates/statefulset.yaml b/deploy/chart/templates/statefulset.yaml index b9beff7..e725686 100644 --- a/deploy/chart/templates/statefulset.yaml +++ b/deploy/chart/templates/statefulset.yaml @@ -18,7 +18,15 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} affinity: + {{- with .Values.nodeAffinity }} + nodeAffinity: + {{- toYaml . | nindent 10 }} + {{- end }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index a3b08dc..98b7e7d 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -5,6 +5,7 @@ namespace: squid # if one pod dies the other keeps serving - no failover window. replicas: 2 nodeSelector: {} +nodeAffinity: {} # Probe tuning. readiness drives Service endpoint membership, so keep it # fast: 5s x 2 = worst-case ~10s to drop a dead endpoint (HTTP cachemgr @@ -29,7 +30,7 @@ images: repository: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/boynux/squid-exporter tag: "v1.13.0" -alpineMirror: "https://mirrors.ustc.edu.cn/alpine/v3.23" +alpineMirror: "https://mirrors.huaweicloud.com/alpine/v3.23" squid: cacheMemory: 512 diff --git a/deploy/values-gy-001.yaml b/deploy/values-gy-001.yaml new file mode 100644 index 0000000..42590f2 --- /dev/null +++ b/deploy/values-gy-001.yaml @@ -0,0 +1,66 @@ +# 本文件是 **gy-001 集群**(openmerlin-guiyang-001)的 values 文件 +# +# 部署时使用(ArgoCD 的 $values 源,或 helm -f): +# helm install squid ./chart -f values-gy-001.yaml -n squid --kubeconfig ~/.kube/gy-001.yaml +# +# 调度:nodeAffinity 绑定唯一 cache 节点 172.16.0.37(amd64,label cache=true,taint cache) +# 必须带 cache toleration,否则无法调度到该节点。 +# +# Environment gy-001: production-grade resources, dual-active replicas +replicas: 2 + +nodeSelector: + kubernetes.io/arch: amd64 + +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - 172.16.0.37 + +tolerations: +- key: cache + operator: Equal + value: "true" + effect: NoSchedule + +squid: + maxObjectSize: 8192 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2 + memory: 4Gi + +registryProxy: + proxyReadTimeout: "300s" + proxySendTimeout: "300s" + proxyConnectReadTimeout: "300s" + sendTimeout: "300s" + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 8 + memory: 16Gi + +persistence: + squidCache: + size: 50Gi + registryCache: + size: 200Gi + +secretDefinition: + enabled: true + vaultPath: secrets/data/ascend/ci + caBundleKey: squid_ca_bundle_v3_prod_pem + caPublicKey: squid_ca_v3_prod_pem + caTruststoreKey: squid_bazel_trust_v3_prod_jks + caNamespaces: + - squid diff --git a/deploy/values-wlcb-001.yaml b/deploy/values-wlcb-001.yaml new file mode 100644 index 0000000..78d5ed0 --- /dev/null +++ b/deploy/values-wlcb-001.yaml @@ -0,0 +1,70 @@ +# 本文件是 **wlcb-001 集群**(乌兰花/乌兰察布-001)的 values 文件 +# +# 部署时使用(ArgoCD 的 $values 源,或 helm -f): +# helm install squid ./chart -f values-wlcb-001.yaml -n squid --kubeconfig ~/.kube/wlcb-001.yaml +# +# 调度:nodeAffinity 绑定 192.168.1.191 + 192.168.1.49(amd64)。 +# 192.168.1.49 带 label cache=true(无 taint);191 无 cache label —— toleration 无副作用。 +# caNamespaces 与线上 4 个命名空间(squid/op-plugin/mindspeed-mm/mindspeed-bridge)同步。 +# +# Environment wlcb-001: production-grade resources, dual-active replicas +replicas: 2 + +nodeSelector: + kubernetes.io/arch: amd64 + +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - 192.168.1.49 + +tolerations: +- key: cache + operator: Equal + value: "true" + effect: NoSchedule + +squid: + maxObjectSize: 8192 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2 + memory: 4Gi + +registryProxy: + proxyReadTimeout: "300s" + proxySendTimeout: "300s" + proxyConnectReadTimeout: "300s" + sendTimeout: "300s" + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 8 + memory: 16Gi + +persistence: + squidCache: + size: 50Gi + registryCache: + size: 200Gi + +secretDefinition: + enabled: true + vaultPath: secrets/data/ascend/ci + caBundleKey: squid_ca_bundle_v3_prod_pem + caPublicKey: squid_ca_v3_prod_pem + caTruststoreKey: squid_bazel_trust_v3_prod_jks + caNamespaces: + - squid + - op-plugin + - mindspeed-mm + - mindspeed-bridge