Repository navigation
Expand file tree
/
Copy pathanalyze_trials.py
More file actions
executable file
Β·452 lines (374 loc) Β· 17.6 KB
/
Copy pathanalyze_trials.py
File metadata and controls
executable file
Β·452 lines (374 loc) Β· 17.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
#!/usr/bin/env python3
"""
analyze_trials.py
Aggregate a multi-trial experiment campaign into mean Β± 95% CI.
Reads:
<root>/trial<N>_<mode>_<cow>/load_tester_output.txt
<root>/trial<N>_<mode>_<cow>/server.log
<root>/trial<N>_<mode>_<cow>/resource.csv
For each (mode, cow_state), computes:
- Cold starts per cycle (C1-Spike / C2-Spike / C3-Spike / C4-Spike), mean Β± 95% CI
- Total cold starts, mean Β± 95% CI
- Predictor inference latency (ΞΌs), p50 / p99
- Average PSS (MB) across the run, mean Β± 95% CI
- Average RSS (MB) across the run, mean Β± 95% CI
Output:
- Markdown summary table to stdout
- <root>/summary.csv (machine-readable summary)
- <root>/per_cycle.csv (per-cycle breakdown for plotting)
Usage:
python3 analyze_trials.py logs/multitrial_<TIMESTAMP>/
CSCI 599: Network Systems for Cloud Computing
University of Southern California
"""
import csv
import os
import re
import sys
from collections import defaultdict
from glob import glob
from statistics import mean, stdev
try:
from scipy.stats import t as student_t
HAS_SCIPY = True
except ImportError:
HAS_SCIPY = False
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Stats helpers
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def ci95(values):
"""
95% confidence interval half-width. Returns (mean, half_width).
Falls back to NaN-friendly values for n<2 or constant series.
"""
if not values:
return (float("nan"), float("nan"))
if len(values) == 1:
return (values[0], float("nan"))
m = mean(values)
s = stdev(values)
if s == 0:
return (m, 0.0)
se = s / (len(values) ** 0.5)
if HAS_SCIPY:
t_crit = student_t.ppf(0.975, df=len(values) - 1)
else:
# Approximation: critical t-values for 95% CI by df.
t_table = {1: 12.71, 2: 4.30, 3: 3.18, 4: 2.78, 5: 2.57,
6: 2.45, 7: 2.36, 8: 2.31, 9: 2.26, 10: 2.23}
t_crit = t_table.get(len(values) - 1, 1.96) # β z for large n
return (m, t_crit * se)
def fmt(m, h, decimals=1):
"""Format 'mean Β± half_ci' in a fixed-width way."""
if h != h: # NaN
return f"{m:.{decimals}f} (n=1)"
return f"{m:.{decimals}f} Β± {h:.{decimals}f}"
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Per-trial parsers
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
CYCLES = ["C1-Spike", "C2-Spike", "C3-Spike", "C4-Spike"]
def parse_load_tester(path):
"""
Parse the per-cycle cold-start counts from load_tester's
'SPIKE COMPARISON' summary table at the end of the output:
================================================================
SPIKE COMPARISON ACROSS CYCLES (primary metric)
================================================================
Metric C1-Spike C2-Spike C3-Spike C4-Spike
---------------------- ---------- ---------- ---------- ----------
Cold starts 0 14 33 0
Warm hits 150 136 117 150
...
Strategy: locate the 'SPIKE COMPARISON' header, then within the next
~20 lines find the row whose label (first non-whitespace token) is
'Cold' (i.e. 'Cold starts'), then extract the 4 trailing integers
matching the 4 spike phases.
Returns {cycle_label: cold_count}; missing cycles map to None.
"""
out = {c: None for c in CYCLES}
if not os.path.exists(path):
return out
with open(path, errors="ignore") as f:
lines = f.readlines()
for i, line in enumerate(lines):
if "SPIKE COMPARISON" not in line:
continue
# Found the table header β scan ~20 following lines for the
# 'Cold starts' row.
for j in range(i, min(i + 25, len(lines))):
stripped = lines[j].lstrip()
if not stripped.startswith("Cold starts"):
continue
# Strip the 'Cold starts' label and pull all integers after it.
nums = re.findall(r"-?\d+", stripped[len("Cold starts"):])
if len(nums) >= 4:
for k, c in enumerate(CYCLES):
out[c] = int(nums[k])
break
break
return out
# Predictor latency lines (added by A3):
# [PredLatency] mode=ewma us=12 rps=20 target=3
_PREDLAT_RE = re.compile(r"\[PredLatency\]\s+mode=\S+\s+us=(\d+)")
def parse_pred_latency_us(path):
"""Returns a list of microsecond samples from server.log."""
out = []
if not os.path.exists(path):
return out
with open(path, errors="ignore") as f:
for line in f:
m = _PREDLAT_RE.search(line)
if m:
out.append(int(m.group(1)))
return out
def parse_resource_csv(path):
"""Returns lists: rss_total (MB), pss_total (MB) β sampled at 1 Hz."""
rss, pss = [], []
if not os.path.exists(path):
return rss, pss
with open(path) as f:
reader = csv.DictReader(f)
for row in reader:
try:
rss.append(float(row.get("rss_total_mb", 0) or 0))
pss.append(float(row.get("pss_total_mb", 0) or 0))
except ValueError:
pass
return rss, pss
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Aggregation
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Directory name patterns this analyzer recognizes:
#
# Main / ablation:
# trial<N>_<mode>_<cow> (run_multi_trial.sh)
# mode β {ewma, ewma_adaptive, reactive, static_15, arima}
# cow β {cow, nocow}
#
# Warmup sweep (multi-trial):
# trial<N>_warmup_<W>s_<mode> (run_sweep.sh --trials N)
#
# Warmup sweep (single-trial, legacy):
# warmup_<W>s_<mode> (run_sweep.sh)
_MAIN_RE = re.compile(r"^trial(\d+)_([a-z0-9_]+?)_(cow|nocow)$")
_SWEEP_M_RE = re.compile(r"^trial(\d+)_warmup_(\d+)s_([a-z_]+)$")
_SWEEP_S_RE = re.compile(r"^warmup_(\d+)s_([a-z_]+)$")
def _extract_metrics(full):
"""Return (cold_dict, total, pred_us_list, rss_list, pss_list)."""
cold = parse_load_tester(os.path.join(full, "load_tester_output.txt"))
total = sum(v for v in cold.values() if v is not None)
pred_us = parse_pred_latency_us(os.path.join(full, "server.log"))
rss, pss = parse_resource_csv(os.path.join(full, "resource.csv"))
return cold, total, pred_us, rss, pss
def _new_bucket():
return {
"cold": [], "total": [],
"pred_us_p50": [], "pred_us_p99": [],
"rss_avg": [], "pss_avg": [],
}
def _push_metrics(bucket, cold, total, pred_us, rss, pss):
bucket["cold"].append(cold)
bucket["total"].append(total)
if pred_us:
pred_us.sort()
bucket["pred_us_p50"].append(pred_us[len(pred_us) // 2])
bucket["pred_us_p99"].append(pred_us[max(0, int(len(pred_us) * 0.99) - 1)])
if rss: bucket["rss_avg"].append(mean(rss))
if pss: bucket["pss_avg"].append(mean(pss))
def collect(root):
"""
Walk the campaign root directory and route each per-trial dir to the
right grouping.
Returns:
{
"kind": "main" | "sweep" | "mixed", # how to render
"main": { (mode, cow): bucket }, # for main/ablation
"sweep": { (W:int, mode): bucket }, # for sweep
}
A bucket is a dict with the same fields as before (cold / total /
pred_us_p50 / pred_us_p99 / rss_avg / pss_avg) β values are lists,
one entry per trial.
"""
main = defaultdict(_new_bucket)
sweep = defaultdict(_new_bucket)
for entry in sorted(os.listdir(root)):
full = os.path.join(root, entry)
if not os.path.isdir(full):
continue
m_main = _MAIN_RE.match(entry)
m_sw_m = _SWEEP_M_RE.match(entry)
m_sw_s = _SWEEP_S_RE.match(entry)
if m_main:
cold, total, pred_us, rss, pss = _extract_metrics(full)
_push_metrics(main[(m_main.group(2), m_main.group(3))],
cold, total, pred_us, rss, pss)
elif m_sw_m:
cold, total, pred_us, rss, pss = _extract_metrics(full)
_push_metrics(sweep[(int(m_sw_m.group(2)), m_sw_m.group(3))],
cold, total, pred_us, rss, pss)
elif m_sw_s:
cold, total, pred_us, rss, pss = _extract_metrics(full)
_push_metrics(sweep[(int(m_sw_s.group(1)), m_sw_s.group(2))],
cold, total, pred_us, rss, pss)
if main and not sweep:
kind = "main"
elif sweep and not main:
kind = "sweep"
elif main and sweep:
kind = "mixed"
else:
kind = "main" # empty β render() will print "no data"
return {"kind": kind, "main": dict(main), "sweep": dict(sweep)}
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Reporting
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
MODES_ORDER = ["static_15", "ewma_adaptive", "ewma", "reactive", "arima"]
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Renderers
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def _bucket_strs(g):
"""Common per-bucket formatted strings (per-cycle CIs, total, PSS, RSS, pred)."""
n = len(g["total"])
cycle_strs = []
for c in CYCLES:
vals = [t.get(c) for t in g["cold"] if t.get(c) is not None]
cycle_strs.append(fmt(*ci95(vals), 1) if vals else "β")
total_str = fmt(*ci95(g["total"]), 1)
pss_str = fmt(*ci95(g["pss_avg"]), 0) if g["pss_avg"] else "β"
rss_str = fmt(*ci95(g["rss_avg"]), 0) if g["rss_avg"] else "β"
if g["pred_us_p50"]:
pred_str = f"{int(mean(g['pred_us_p50']))} / {int(mean(g['pred_us_p99']))}"
else:
pred_str = "β"
return n, cycle_strs, total_str, pss_str, rss_str, pred_str
def render_main(main_groups):
"""Print main / ablation summary tables (one per CoW state)."""
has_ablation = any(cow == "nocow" for (_, cow) in main_groups.keys())
cols = ["CoW=ON"] + (["CoW=OFF"] if has_ablation else [])
for col in cols:
cow_key = "cow" if col == "CoW=ON" else "nocow"
print(f"## {col}\n")
print("| Mode | C1-Spike | C2-Spike | C3-Spike | C4-Spike | Total | n | PSS avg (MB) | RSS avg (MB) | pred ΞΌs (p50 / p99) |")
print("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|")
for mode in MODES_ORDER:
g = main_groups.get((mode, cow_key))
if not g:
continue
n, cs, total_str, pss_str, rss_str, pred_str = _bucket_strs(g)
print(f"| {mode} | {cs[0]} | {cs[1]} | {cs[2]} | {cs[3]} | "
f"**{total_str}** | {n} | {pss_str} | {rss_str} | {pred_str} |")
print()
def render_sweep(sweep_groups):
"""Print warmup-sweep summary: one row per (W, mode), sorted by W asc."""
print("## Warmup sweep (Fixed vs Adaptive CUSUM across inter-burst intervals)\n")
print("| W (s) | Mode | C1-Spike | C2-Spike | C3-Spike | C4-Spike | Total | n |")
print("|---:|---|---:|---:|---:|---:|---:|---:|")
for w in sorted({W for (W, _m) in sweep_groups.keys()}):
for mode in ["ewma", "ewma_adaptive"]: # fixed first, then adaptive
g = sweep_groups.get((w, mode))
if not g:
continue
n, cs, total_str, _pss, _rss, _pred = _bucket_strs(g)
print(f"| {w} | {mode} | {cs[0]} | {cs[1]} | {cs[2]} | {cs[3]} | "
f"**{total_str}** | {n} |")
print()
def render_markdown(groups):
"""Print a markdown summary appropriate for the campaign kind."""
print()
print("# Multi-trial summary")
print()
if groups["kind"] in ("main", "mixed") and groups["main"]:
render_main(groups["main"])
if groups["kind"] in ("sweep", "mixed") and groups["sweep"]:
render_sweep(groups["sweep"])
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# CSV writers
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def _bucket_row(g):
"""Common numeric row for a single bucket β used by both writers."""
n = len(g["total"])
if n == 0:
return None
mt, ht = ci95(g["total"])
mp, hp = ci95(g["pss_avg"])
mr, hr = ci95(g["rss_avg"])
row = {
"n_trials": n,
"total_mean": mt, "total_ci95": ht,
"pss_avg_mean": mp, "pss_avg_ci95": hp,
"rss_avg_mean": mr, "rss_avg_ci95": hr,
"pred_us_p50_mean": mean(g["pred_us_p50"]) if g["pred_us_p50"] else "",
"pred_us_p99_mean": mean(g["pred_us_p99"]) if g["pred_us_p99"] else "",
}
for c in CYCLES:
vals = [t.get(c) for t in g["cold"] if t.get(c) is not None]
if vals:
m, h = ci95(vals)
row[f"{c}_mean"] = m
row[f"{c}_ci95"] = h
return row
def write_summary_csv(groups, out_path):
"""One CSV row per group (main: per (mode,cow); sweep: per (W,mode))."""
rows = []
for (mode, cow), g in groups["main"].items():
r = _bucket_row(g)
if r:
r["mode"] = mode; r["cow"] = cow
rows.append(r)
for (w, mode), g in groups["sweep"].items():
r = _bucket_row(g)
if r:
r["mode"] = mode; r["warmup_s"] = w
rows.append(r)
if not rows:
return
fields = sorted({k for r in rows for k in r.keys()})
with open(out_path, "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=fields)
w.writeheader()
for r in rows:
w.writerow(r)
def write_per_cycle_csv(groups, out_path):
"""Per-trial per-cycle breakdown for plotting (long format)."""
with open(out_path, "w", newline="") as f:
w = csv.writer(f)
w.writerow(["mode", "cow_or_warmup_s", "trial_idx", "cycle", "cold"])
for (mode, cow), g in groups["main"].items():
for trial_idx, cold_dict in enumerate(g["cold"], start=1):
for c in CYCLES:
val = cold_dict.get(c)
if val is not None:
w.writerow([mode, cow, trial_idx, c, val])
for (warmup_s, mode), g in groups["sweep"].items():
for trial_idx, cold_dict in enumerate(g["cold"], start=1):
for c in CYCLES:
val = cold_dict.get(c)
if val is not None:
w.writerow([mode, warmup_s, trial_idx, c, val])
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# Main
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def main():
if len(sys.argv) != 2:
print(__doc__)
sys.exit(1)
root = sys.argv[1].rstrip("/")
if not os.path.isdir(root):
print(f"Error: not a directory: {root}", file=sys.stderr)
sys.exit(1)
if not HAS_SCIPY:
print("Note: scipy not installed; using approximate t-table for 95% CI.",
file=sys.stderr)
groups = collect(root)
if not (groups["main"] or groups["sweep"]):
print(f"No recognised trial directories found under {root}", file=sys.stderr)
sys.exit(2)
render_markdown(groups)
summary_path = os.path.join(root, "summary.csv")
per_cycle_path = os.path.join(root, "per_cycle.csv")
write_summary_csv(groups, summary_path)
write_per_cycle_csv(groups, per_cycle_path)
print(f"Wrote: {summary_path}")
print(f"Wrote: {per_cycle_path}")
if __name__ == "__main__":
main()