Store results one file per framework

.github/workflows/benchmark-pr.yml

@@ -134,6 +134,8 @@ jobs:
 
       - name: Commit saved results
         if: inputs.save == 'true'
+        env:
+          FRAMEWORK: ${{ inputs.framework }}
         run: |
           PR_DATA=$(gh api "/repos/${{ github.repository }}/pulls/${{ inputs.pr }}" --jq '.head.ref + " " + .head.repo.full_name')
           PR_BRANCH=$(echo "$PR_DATA" | awk '{print $1}')
@@ -143,14 +145,11 @@ jobs:
           git config user.email "github-actions[bot]@users.noreply.github.com"
           find site/static/logs -name "${{ inputs.framework }}.log" -exec git add -f {} +
           git add -f site/data/frameworks.json site/data/current.json 2>/dev/null || true
-          # Only add leaderboard files for the profiles that were actually benchmarked
-          for f in results/*/*/${{ inputs.framework }}.json; do
-            [ -f "$f" ] || continue
-            dir=$(dirname "$f")
-            conns=$(basename "$dir")
-            prof=$(basename "$(dirname "$dir")")
-            git add -f "site/data/${prof}-${conns}.json" 2>/dev/null || true
-          done
+          # Results are one file per framework (#751), so a run only ever stages
+          # its own — two PRs benchmarking different frameworks no longer touch
+          # the same file. The filename is the slugified display_name.
+          SLUG=$(python3 -c "import json,re,sys;n=json.load(open('frameworks/%s/meta.json'%sys.argv[1])).get('display_name',sys.argv[1]);print((re.sub(r'[^A-Za-z0-9._-]+','-',n).strip('-') or 'unnamed').lower())" "$FRAMEWORK")
+          git add -f "site/data/results/${SLUG}.json" 2>/dev/null || true
           if git diff --cached --quiet; then
             echo "No results to commit"
           else

.gitignore

@@ -1,4 +1,6 @@
-results/
+# Raw benchmark output at the repo root. Anchored: site/data/results/ holds
+# the published per-framework result files and must stay tracked.
+/results/
 site/public/
 site/resources/
 

scripts/archive.sh

@@ -87,16 +87,25 @@ import json, glob, os, sys
 site_data = sys.argv[1]
 round_file = sys.argv[2]
 
+# Rounds keep their existing shape - {"<profile>-<conns>": [row, ...]} - so
+# archived rounds stay readable. Results now live one file per framework
+# (#751), so regroup them back into per-profile arrays here.
 bundle = {}
-for f in sorted(glob.glob(os.path.join(site_data, '*.json'))):
-    name = os.path.basename(f)
-    if name in ('frameworks.json', 'langcolors.json'):
+for f in sorted(glob.glob(os.path.join(site_data, 'results', '*.json'))):
+    try:
+        with open(f) as fh:
+            entry = json.load(fh)
+    except Exception:
         continue
-    if name.startswith('rounds'):
-        continue
-    key = os.path.splitext(name)[0]
-    with open(f) as fh:
-        bundle[key] = json.load(fh)
+    for key, row in (entry.get('results') or {}).items():
+        bundle.setdefault(key, []).append(row)
+for key in bundle:
+    bundle[key].sort(key=lambda r: (r.get('framework') or '').lower())
+
+current = os.path.join(site_data, 'current.json')
+if os.path.exists(current):
+    with open(current) as fh:
+        bundle['current'] = json.load(fh)
 
 # Include frameworks metadata
 fw_path = os.path.join(site_data, 'frameworks.json')

scripts/compare.sh

@@ -64,7 +64,7 @@ SITE_DATA="$ROOT_DIR/site/data"
 
 # Collect all profiles with results for this framework
 python3 -c "
-import json, os, sys, glob
+import json, os, re, sys, glob
 
 framework = sys.argv[1]
 display_name = sys.argv[2]
@@ -106,21 +106,24 @@ if not new_results:
     print(f'No results found for \`{framework}\`')
     sys.exit(0)
 
-# Find old results from site/data (published on main)
+# Find old results from site/data (published on main). Results are one file
+# per framework keyed by <profile>-<conns> (#751), so this is a single read.
+def _slug(name):
+    return (re.sub(r'[^A-Za-z0-9._-]+', '-', name).strip('-') or 'unnamed').lower()
+
 old_results = {}
-for key, new_data in new_results.items():
-    profile, conns = key.split('/')
-    site_file = f'{site_data}/{profile}-{conns}.json'
-    if os.path.exists(site_file):
-        try:
-            with open(site_file) as f:
-                entries = json.load(f)
-            for entry in entries:
-                if entry.get('framework') == baseline_name:
-                    old_results[key] = entry
-                    break
-        except:
-            pass
+baseline_file = f'{site_data}/results/{_slug(baseline_name)}.json'
+if os.path.exists(baseline_file):
+    try:
+        with open(baseline_file) as f:
+            published = (json.load(f).get('results') or {})
+        for key in new_results:
+            profile, conns = key.split('/')
+            entry = published.get(f'{profile}-{conns}')
+            if entry:
+                old_results[key] = entry
+    except:
+        pass
 
 # Format helpers
 def fmt_num(n):

scripts/gen_new_leaderboard_data.py

@@ -123,6 +123,37 @@
 }
 
 
+RESULTS: dict[str, list] = {}
+
+
+def load_results():
+    """Index site/data/results/*.json as {"<profile>-<conns>": [row, ...]}.
+
+    Results used to live in one array per profile-conns, which meant every
+    framework's PR wrote the same files and collided (#751). They are now one
+    file per framework; this rebuilds the per-profile view the rest of the
+    generator expects.
+
+    Rows are sorted by framework name because that is the order the flat files
+    were written in, and the emitted data.js must not churn.
+    """
+    idx: dict[str, list] = {}
+    rdir = DATA / "results"
+    if not rdir.is_dir():
+        return idx
+    for f in sorted(rdir.glob("*.json")):
+        try:
+            entry = json.loads(f.read_text(encoding="utf-8"))
+        except Exception as e:
+            print(f"[warn] {f.name}: {e}")
+            continue
+        for key, row in (entry.get("results") or {}).items():
+            idx.setdefault(key, []).append(row)
+    for key in idx:
+        idx[key].sort(key=lambda r: (r.get("framework") or "").lower())
+    return idx
+
+
 def load(name):
     p = DATA / name
     if not p.exists():
@@ -514,6 +545,8 @@ def build_rounds():
 
 
 def main():
+    global RESULTS
+    RESULTS = load_results()
     frameworks = load("frameworks.json") or {}
     langcolors = load("langcolors.json") or {}
     current = load("current.json") or {}
@@ -533,7 +566,7 @@ def main():
         for pid, label, blurb, explorer, scored, s, es in entries:
             present = []
             for c in explorer:
-                rows = load(f"{pid}-{c}.json")
+                rows = RESULTS.get(f"{pid}-{c}")
                 if not rows:
                     continue
                 trimmed = []

scripts/migrate_results_layout.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""One-off: split site/data/<profile>-<conns>.json into per-framework files.
+
+Every framework's results used to live in 52 shared arrays, so two pull
+requests saving results touched the same files and conflicted (#751). After
+this each framework owns exactly one file, and concurrent runs cannot collide.
+
+    site/data/baseline-512.json      [ {actix...}, {hyper...}, ... ]   ->
+    site/data/results/actix.json     { framework, results: { "baseline-512": {...} } }
+
+Rows are copied verbatim, so the generated data.js is unchanged. Run once;
+rebuild_site_data.py writes the new layout from then on.
+"""
+
+from __future__ import annotations
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Files in site/data that are not per-profile result arrays.
+NON_RESULT = {"frameworks.json", "current.json", "langcolors.json"}
+
+
+def slug(name: str) -> str:
+    """Filename for a display name. Two entries ('aspnet-minimal + nginx' and
+    '+ caddy') contain spaces and a plus, so names cannot be used directly."""
+    s = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("-")
+    return s.lower() or "unnamed"
+
+
+def collect(site_data: Path) -> dict[str, dict]:
+    """{slug: {framework, results: {profile-conns: row}}} from the flat files."""
+    out: dict[str, dict] = {}
+    for f in sorted(site_data.glob("*.json")):
+        if f.name in NON_RESULT:
+            continue
+        try:
+            rows = json.loads(f.read_text(encoding="utf-8"))
+        except Exception as e:
+            print(f"[warn] skipping {f.name}: {e}", file=sys.stderr)
+            continue
+        if not isinstance(rows, list):
+            continue
+        key = f.stem                      # e.g. "baseline-512"
+        for row in rows:
+            if not isinstance(row, dict):
+                continue
+            name = row.get("framework")
+            if not name:
+                continue
+            entry = out.setdefault(slug(name), {"framework": name, "results": {}})
+            if entry["framework"] != name:
+                print(f"[warn] slug collision: {entry['framework']!r} vs {name!r}", file=sys.stderr)
+            entry["results"][key] = row
+    return out
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--root", default=str(Path(__file__).resolve().parent.parent))
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    site_data = Path(args.root) / "site" / "data"
+    results_dir = site_data / "results"
+
+    frameworks = collect(site_data)
+    n_rows = sum(len(e["results"]) for e in frameworks.values())
+    print(f"{len(frameworks)} frameworks, {n_rows} rows")
+
+    if args.dry_run:
+        for s, e in sorted(frameworks.items())[:5]:
+            print(f"  {s}.json  <- {e['framework']} ({len(e['results'])} rows)")
+        return
+
+    results_dir.mkdir(parents=True, exist_ok=True)
+    for s, entry in sorted(frameworks.items()):
+        entry["results"] = dict(sorted(entry["results"].items()))
+        (results_dir / f"{s}.json").write_text(json.dumps(entry, indent=2) + "\n", encoding="utf-8")
+
+    removed = 0
+    for f in sorted(site_data.glob("*.json")):
+        if f.name in NON_RESULT:
+            continue
+        f.unlink()
+        removed += 1
+    print(f"wrote {len(frameworks)} files to {results_dir.relative_to(Path(args.root))}, "
+          f"removed {removed} flat files")
+
+
+if __name__ == "__main__":
+    main()