Skip to content

Record the cgroup memory breakdown on every result #1045

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MDA2AV wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Record the cgroup memory breakdown on every result #1045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion scripts/benchmark-lite.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ run_one() {
best_output="$output"
best_cpu="$STATS_AVG_CPU"
best_mem="$STATS_PEAK_MEM"
BEST_MEM_DETAIL="$STATS_MEM_DETAIL"
BEST_M=()
for k in "${!m[@]}"; do BEST_M[$k]="${m[$k]}"; done
fi
Expand Down Expand Up @@ -321,6 +322,16 @@ save_result() {

# Composite-score support — api-4/16 + gateway-64 need per-template splits
# or the website scores them as 0. See save_result in benchmark.sh.
# cgroup composition at peak memory, in MiB. Recorded for #1015: the
# `memory` field above is docker's number (memory.current minus
# inactive_file) and cannot show whether it is application memory, page
# cache the kernel kept active, or socket buffers. Nothing reads this yet.
local mem_detail_extra=""
if [ -n "${BEST_MEM_DETAIL:-}" ]; then
mem_detail_extra=",
\"memory_detail\": {${BEST_MEM_DETAIL}}"
fi

local tpl_extra=""
if [ "$profile" = "api-4" ] || [ "$profile" = "api-16" ]; then
tpl_extra=",
Expand Down Expand Up @@ -359,7 +370,7 @@ save_result() {
"status_2xx": ${BEST_M[status_2xx]:-0},
"status_3xx": ${BEST_M[status_3xx]:-0},
"status_4xx": ${BEST_M[status_4xx]:-0},
"status_5xx": ${BEST_M[status_5xx]:-0}${tpl_extra}
"status_5xx": ${BEST_M[status_5xx]:-0}${mem_detail_extra}${tpl_extra}
}
EOF
info "saved results/$profile/$CONNS/${FRAMEWORK}.json"
Expand Down
14 changes: 13 additions & 1 deletion scripts/benchmark.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ run_one() {
# even if its rps is 0 (ws-echo, zero-traffic regressions). Without this,
# BEST_M would carry stale metrics from a previous profile.
local best_rps=-1 best_output="" best_cpu="0%" best_mem="0MiB" best_breakdown=""
BEST_MEM_DETAIL=""
BEST_M=()
local run

Expand Down Expand Up @@ -317,6 +318,7 @@ run_one() {
best_cpu="$STATS_AVG_CPU"
best_mem="$STATS_PEAK_MEM"
best_breakdown="$STATS_BREAKDOWN"
BEST_MEM_DETAIL="$STATS_MEM_DETAIL"
BEST_M=()
for k in "${!m[@]}"; do BEST_M[$k]="${m[$k]}"; done
fi
Expand Down Expand Up @@ -398,6 +400,16 @@ save_result() {
\"cpu_breakdown\": \"$best_breakdown\""
fi

# cgroup composition at peak memory, in MiB. Recorded for #1015: the
# `memory` field above is docker's number (memory.current minus
# inactive_file) and cannot show whether it is application memory, page
# cache the kernel kept active, or socket buffers. Nothing reads this yet.
local mem_detail_extra=""
if [ -n "${BEST_MEM_DETAIL:-}" ]; then
mem_detail_extra=",
\"memory_detail\": {${BEST_MEM_DETAIL}}"
fi

local tpl_extra=""
if [ "$profile" = "api-4" ] || [ "$profile" = "api-16" ]; then
tpl_extra=",
Expand Down Expand Up @@ -449,7 +461,7 @@ save_result() {
"status_2xx": ${BEST_M[status_2xx]:-0},
"status_3xx": ${BEST_M[status_3xx]:-0},
"status_4xx": ${BEST_M[status_4xx]:-0},
"status_5xx": ${BEST_M[status_5xx]:-0}${tpl_extra}${cpu_extra}
"status_5xx": ${BEST_M[status_5xx]:-0}${mem_detail_extra}${tpl_extra}${cpu_extra}
}
EOF
info "saved results/$profile/$CONNS/${FRAMEWORK}.json"
Expand Down
82 changes: 82 additions & 0 deletions scripts/lib/stats.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,43 @@
# stats_start <container...> # starts background collector
# stats_stop # stops, fills STATS_AVG_CPU / STATS_PEAK_MEM
# # and (multi-container) STATS_BREAKDOWN
# # and STATS_MEM_DETAIL
#
# `docker stats` reports memory.current minus inactive_file, so it already
# excludes most page cache - but that single number says nothing about what
# the memory actually is. #1015 asks whether the published figures are really
# the application's memory; answering it needs the composition, not another
# opinion. So alongside the existing number we record the cgroup's own
# accounting (anon, file, sock, slab, kernel) at the moment of peak usage and
# store it on the result row for later analysis. Nothing consumes it yet: the
# leaderboard still reads `memory`.

STATS_PID=""
STATS_LOG=""
STATS_CG_LOG=""
STATS_AVG_CPU="0%"
STATS_PEAK_MEM="0MiB"
STATS_BREAKDOWN=""
STATS_MEM_DETAIL=""

# Fields worth keeping from cgroup v2 memory.stat. anon is application memory,
# file is page cache (active_file is the part docker still counts), sock is
# kernel socket buffers - which at 4096 connections is not a rounding error -
# and slab/kernel are kernel structures charged to the container.
STATS_CG_FIELDS="anon file active_file inactive_file sock slab kernel"

# Resolve a container's cgroup directory. Layout varies with the cgroup driver
# (systemd vs cgroupfs) and version, so try the known shapes and give up
# quietly - this is supplementary data, never a reason to fail a run.
_cg_dir() {
local id="$1" p
for p in "/sys/fs/cgroup/system.slice/docker-$id.scope" \
"/sys/fs/cgroup/docker/$id" \
"/sys/fs/cgroup/memory/docker/$id"; do
[ -r "$p/memory.stat" ] && { echo "$p"; return 0; }
done
return 1
}

# Start a background poller. Accepts one or more container names. Each
# sample writes one line per container, tagged with a snapshot counter so
Expand All @@ -26,7 +57,17 @@ STATS_BREAKDOWN=""
# Log line format: <snap> <container-name> <cpu%> <mem-MiB>
stats_start() {
STATS_LOG=$(mktemp)
STATS_CG_LOG=$(mktemp)
local containers=("$@")

# Resolve cgroup paths once; container ids don't change mid-run.
local _cg_paths=() _c _id _dir
for _c in "${containers[@]}"; do
_id=$(docker inspect -f '{{.Id}}' "$_c" 2>/dev/null) || continue
_dir=$(_cg_dir "$_id") || continue
_cg_paths+=("$_dir")
done

(
local snap=0
while true; do
Expand All @@ -49,6 +90,16 @@ stats_start() {
else if (unit == "B") mem_mib /= (1024 * 1024)
printf "%s %s %.2f %.2f\n", snap, name, cpu, mem_mib
}'

# cgroup accounting for the same snapshot, summed across containers
for _d in "${_cg_paths[@]}"; do
awk -v snap="$snap" -v want="$STATS_CG_FIELDS" '
BEGIN { n = split(want, w, " "); for (i = 1; i <= n; i++) keep[w[i]] = 1 }
keep[$1] { printf "%s %s %s\n", snap, $1, $2 }
' "$_d/memory.stat" 2>/dev/null
awk -v snap="$snap" '{ printf "%s current %s\n", snap, $1 }' \
"$_d/memory.current" 2>/dev/null
done >>"$STATS_CG_LOG"
done
) >"$STATS_LOG" 2>/dev/null &
STATS_PID=$!
Expand Down Expand Up @@ -90,6 +141,37 @@ stats_stop() {
}
' "$STATS_LOG")

# ── cgroup composition at the peak snapshot ─────────────────────────
# Reported as MiB per field. The snapshot is chosen by total memory so
# the breakdown describes the same moment STATS_PEAK_MEM reports,
# rather than being a max-per-field mixture of different instants.
STATS_MEM_DETAIL=""
if [ -s "${STATS_CG_LOG:-/dev/null}" ]; then
STATS_MEM_DETAIL=$(awk '
{ v[$1 SUBSEP $2] += $3; if (!($1 in seen)) { seen[$1]=1 } }
END {
best = ""; bestv = -1
for (k in v) {
split(k, a, SUBSEP)
if (a[2] == "current" && v[k] > bestv) { bestv = v[k]; best = a[1] }
}
if (best == "") exit
n = split("current anon file active_file inactive_file sock slab kernel", f, " ")
out = ""
for (i = 1; i <= n; i++) {
key = best SUBSEP f[i]
if (key in v) {
if (out != "") out = out ","
out = out sprintf("\"%s\":%.1f", f[i], v[key] / 1048576)
}
}
print out
}
' "$STATS_CG_LOG")
fi
rm -f "${STATS_CG_LOG:-}" 2>/dev/null || true
STATS_CG_LOG=""

# ── Per-container breakdown — average CPU and peak mem per container,
# rendered as "proxy: 4200% 1.2GiB | server: 1200% 512MiB". Skipped
# entirely when only one container was sampled (the breakdown would
Expand Down