/
/
/
1"""
2Report rendering and baseline comparison for the performance benchmark suite.
3
4Works on the JSON documents produced by run_benchmark.py. All metrics carry their
5unit in the key name (cpu_seconds, rss_mb, p95_ms, payload_kb, ...) and for every
6unit in this suite a higher value is worse.
7"""
8
9from __future__ import annotations
10
11from typing import Any
12
13# Relative regression threshold applied to every compared metric,
14# unless overridden per metric below.
15DEFAULT_REL_THRESHOLD = 0.15
16
17# Per-metric relative thresholds, matched on the full dotted metric path first,
18# then on the trailing path segment. Values express the allowed relative increase.
19PER_METRIC_REL_THRESHOLDS: dict[str, float] = {
20 # p95 latency is inherently noisier than the median
21 "p95_ms": 0.30,
22 # wall times absorb disk/scheduler noise; the cpu_seconds metrics are the signal
23 "wall_seconds": 0.30,
24 # scheduling jitter; only flag when it explodes
25 "max_loop_lag_ms": 1.0,
26 # short capture window on small absolute numbers; only flag structural changes
27 "streaming.python_cpu_seconds": 0.5,
28 "streaming.ffmpeg_cpu_seconds": 0.5,
29}
30
31# Absolute floors per unit suffix: a delta smaller than this is never flagged,
32# so tiny absolute changes on near-zero metrics don't trip the relative check.
33ABS_FLOORS: dict[str, float] = {
34 "_ms": 5.0,
35 "_seconds": 0.25,
36 "_mb": 8.0,
37 "_kb": 2.0,
38}
39
40# Per-metric absolute floors (matched like the relative overrides), for metrics whose
41# baseline sits so close to zero that scheduler jitter dwarfs any relative threshold.
42PER_METRIC_ABS_FLOORS: dict[str, float] = {
43 # single-digit-ms baselines routinely jitter to ~25ms; real lag problems are 100ms+
44 "max_loop_lag_ms": 50.0,
45}
46
47
48def flatten_metrics(scenarios: dict[str, Any]) -> dict[str, float]:
49 """Flatten the scenarios tree into {dotted.path: value} for all numeric metrics."""
50 flat: dict[str, float] = {}
51
52 def _walk(node: dict[str, Any], prefix: str) -> None:
53 for key, value in node.items():
54 path = f"{prefix}.{key}" if prefix else key
55 if isinstance(value, dict):
56 _walk(value, path)
57 elif isinstance(value, (int, float)) and not isinstance(value, bool):
58 flat[path] = float(value)
59
60 _walk(scenarios, "")
61 return flat
62
63
64def compare_reports(baseline: dict[str, Any], current: dict[str, Any]) -> tuple[list[str], int]:
65 """
66 Compare two benchmark reports metric-by-metric.
67
68 :param baseline: The baseline report document.
69 :param current: The report document to check for regressions.
70 :return: Tuple of (human-readable lines, number of regressed metrics).
71 """
72 base_flat = flatten_metrics(baseline.get("scenarios", {}))
73 cur_flat = flatten_metrics(current.get("scenarios", {}))
74 lines: list[str] = []
75 regressions = 0
76
77 lines.append(f"{'metric':60s} {'baseline':>12s} {'current':>12s} {'delta':>9s} status")
78 for path in sorted(set(base_flat) | set(cur_flat)):
79 base_val = base_flat.get(path)
80 cur_val = cur_flat.get(path)
81 if base_val is None or cur_val is None:
82 lines.append(
83 f"{path:60s} {_fmt(base_val):>12s} {_fmt(cur_val):>12s} {'-':>9s} "
84 + ("added" if base_val is None else "removed")
85 )
86 continue
87 delta = cur_val - base_val
88 rel = delta / base_val if base_val else 0.0
89 status = "ok"
90 if _is_regression(path, base_val, cur_val):
91 status = "REGRESSED"
92 regressions += 1
93 elif rel <= -DEFAULT_REL_THRESHOLD and abs(delta) >= _abs_floor(path):
94 status = "improved"
95 rel_str = f"{rel:+.1%}" if base_val else "n/a"
96 lines.append(f"{path:60s} {base_val:12.2f} {cur_val:12.2f} {rel_str:>9s} {status}")
97
98 lines.append("")
99 lines.append(
100 f"{regressions} regressed metric(s)" if regressions else "no regressions beyond thresholds"
101 )
102 return lines, regressions
103
104
105def render_markdown(report: dict[str, Any]) -> str:
106 """Render a benchmark report as human-readable markdown tables."""
107 out: list[str] = []
108 env = report.get("env", {})
109 meta = report.get("meta", {})
110 mode = "quick" if meta.get("quick") else "full"
111 out.append(f"# Music Assistant perf benchmark ({mode})")
112 out.append("")
113 out.append(
114 f"`{env.get('git_sha', '?')[:10]}`{' (dirty)' if env.get('dirty') else ''} · "
115 f"python {env.get('python', '?')} · {env.get('cpu', '?')} · "
116 f"{meta.get('generated_at', '?')}"
117 )
118
119 for name, metrics in report.get("scenarios", {}).items():
120 out.append("")
121 out.append(f"## {name}")
122 out.append("")
123 if name == "api_bench":
124 out.append("| command | median_ms | p95_ms | payload_kb | items |")
125 out.append("|---|---:|---:|---:|---:|")
126 for cmd, values in metrics.items():
127 out.append(
128 f"| {cmd} | {values['median_ms']} | {values['p95_ms']} "
129 f"| {values['payload_kb']} | {values['items']} |"
130 )
131 continue
132 out.append("| metric | value |")
133 out.append("|---|---:|")
134 for key, value in metrics.items():
135 if isinstance(value, list):
136 continue
137 out.append(f"| {key} | {value} |")
138 if importtime_top := metrics.get("importtime_top"):
139 out.append("")
140 out.append("Slowest imports (self time):")
141 out.append("")
142 out.append("| module | self_ms | cumulative_ms |")
143 out.append("|---|---:|---:|")
144 for row in importtime_top:
145 out.append(f"| {row['module']} | {row['self_ms']} | {row['cumulative_ms']} |")
146
147 if yappi_top := report.get("yappi_top"):
148 out.append("")
149 out.append("## CPU hotspots (yappi, profiled pass)")
150 for scenario, rows in yappi_top.items():
151 out.append("")
152 out.append(f"### {scenario}")
153 out.append("")
154 out.append("| function | ncall | tsub_ms | ttot_ms |")
155 out.append("|---|---:|---:|---:|")
156 for row in rows[:5]:
157 out.append(
158 f"| {row['name']} | {row['ncall']} | {row['tsub_ms']} | {row['ttot_ms']} |"
159 )
160
161 out.append("")
162 return "\n".join(out)
163
164
165def _abs_floor(path: str) -> float:
166 """Return the absolute delta floor for a metric based on its unit suffix."""
167 if path in PER_METRIC_ABS_FLOORS:
168 return PER_METRIC_ABS_FLOORS[path]
169 leaf = path.rsplit(".", 1)[-1]
170 if leaf in PER_METRIC_ABS_FLOORS:
171 return PER_METRIC_ABS_FLOORS[leaf]
172 for suffix, floor in ABS_FLOORS.items():
173 if path.endswith(suffix):
174 return floor
175 return 0.0
176
177
178def _rel_threshold(path: str) -> float:
179 """Return the relative regression threshold for a metric path."""
180 if path in PER_METRIC_REL_THRESHOLDS:
181 return PER_METRIC_REL_THRESHOLDS[path]
182 leaf = path.rsplit(".", 1)[-1]
183 return PER_METRIC_REL_THRESHOLDS.get(leaf, DEFAULT_REL_THRESHOLD)
184
185
186def _is_regression(path: str, base_val: float, cur_val: float) -> bool:
187 """Whether current exceeds baseline beyond the threshold (higher is always worse)."""
188 delta = cur_val - base_val
189 if delta <= 0 or abs(delta) < _abs_floor(path):
190 return False
191 if base_val == 0:
192 return True
193 return (delta / base_val) > _rel_threshold(path)
194
195
196def _fmt(value: float | None) -> str:
197 """Format an optional numeric value for the comparison table."""
198 return "-" if value is None else f"{value:.2f}"
199