Coverage for src/ai_jury/doctor.py: 98%
161 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-26 23:18 +0000
1"""Local diagnostics for the agent review jury (``jury --doctor``).
3The ``--doctor`` command reports local readiness and common configuration
4problems. Its output is intentionally SAFE to share:
6- It includes tool/Python/OS versions, a redacted config summary, agent
7 availability (which agent CLIs are on PATH), each agent's detected CLI
8 version and capability summary, and detected config warnings.
9- It NEVER includes the raw diff under review or any agent output.
10- Secret-like values in the config summary are redacted via
11 :func:`ai_jury.redaction.redact`.
13This project collects and transmits NO telemetry. Diagnostics are built
14locally and only written where you explicitly ask (stdout, or ``--write``).
15"""
17from __future__ import annotations
19import platform
20import shutil
21import sys
22import tomllib
23from pathlib import Path
25from . import __version__
26from .adapters import make_adapter
27from .config import load_config, ConfigError
28from .redaction import redact, redact_url_userinfo
31def _redact_value(value):
32 """Redact a single config value if it looks secret-like.
34 ``redact`` operates on text and returns ``(text, count)``; non-string
35 values are returned unchanged.
36 """
37 if isinstance(value, str):
38 return redact(value)[0]
39 return value
42def _detect_capabilities(spec):
43 """Best-effort capability/version probe for one agent spec.
45 Uses the real adapter (NOT the mock) so doctor reports actual installed
46 versions, but guards against any failure: an unavailable CLI just reports
47 ``status="unavailable"`` and a crashing probe degrades to ``unknown_version``.
48 This must stay fast (short subprocess timeout) and never crash doctor.
49 """
50 try:
51 adapter = make_adapter(spec)
52 return adapter.detect_capabilities()
53 except Exception as exc: # noqa: BLE001 - diagnostics must never crash
54 return {
55 "version": None,
56 "supports_headless": None,
57 "supports_model_selection": None,
58 "raw_version_output": "",
59 "status": "unknown_version",
60 "warnings": [f"capability probe raised: {redact(str(exc))[0]}"],
61 }
64def _is_available(spec) -> bool:
65 """Whether an agent is reachable, via its adapter's own check.
67 Uses ``adapter.available()`` rather than ``shutil.which`` so a local/HTTP
68 agent (issue #43), which has no ``command`` and probes its endpoint instead,
69 is reported correctly. Guarded — any failure reads as unavailable.
70 """
71 try:
72 return make_adapter(spec).available()
73 except Exception: # noqa: BLE001 - diagnostics must never crash
74 return False
77def _resolved_command(spec):
78 """Absolute path a CLI agent's command resolves to on PATH (issue #296).
80 Lets an operator verify *which* binary will run (a poisoned PATH could
81 resolve a bare name to a shim). None for a local/HTTP agent (no command) or
82 when nothing is found on PATH.
83 """
84 command = getattr(spec, "command", "") or ""
85 vendor = (getattr(spec, "vendor", "") or "").lower()
86 if not command or vendor == "local":
87 return None
88 try:
89 return shutil.which(command)
90 except Exception: # noqa: BLE001 - diagnostics must never crash
91 return None
94def _agent_entry(spec):
95 caps = _detect_capabilities(spec)
96 return {
97 "name": _redact_value(spec.name),
98 "command": _redact_value(spec.command),
99 "resolved": _resolved_command(spec),
100 "vendor": _redact_value(spec.vendor),
101 "available": _is_available(spec),
102 "version": _redact_value(caps.get("version")),
103 "capabilities": {
104 "supports_headless": caps.get("supports_headless"),
105 "supports_model_selection": caps.get("supports_model_selection"),
106 "status": caps.get("status"),
107 },
108 "capability_warnings": [_redact_value(w) for w in caps.get("warnings", [])],
109 }
112def _config_summary(cfg):
113 """Build a redacted, secret-free summary of the loaded config."""
114 return {
115 "rounds": cfg.rounds,
116 "chair": _redact_value(cfg.chair),
117 "context_mode": _redact_value(cfg.context.mode),
118 "enabled_agents": [_redact_value(a.name) for a in cfg.enabled_agents],
119 }
122# Hosted-API vendors (issue #430/#432): no `command`/`endpoint`, so neither
123# the "local" nor the "CLI on PATH" branch below is the right diagnosis when
124# one is unavailable.
125_HOSTED_API_VENDORS = ("anthropic-api", "openai-api", "google-api")
128def _detect_warnings(cfg) -> list[str]:
129 """Best-effort config sanity checks reported to the user."""
130 warnings: list[str] = []
131 if not cfg.agents:
132 warnings.append("no agents are configured")
133 enabled = cfg.enabled_agents
134 if cfg.agents and not enabled:
135 warnings.append("all configured agents are disabled")
136 names = {a.name for a in cfg.agents}
137 if cfg.chair not in names:
138 warnings.append(f"chair '{_redact_value(cfg.chair)}' does not match any configured agent")
139 for agent in enabled:
140 if _is_available(agent):
141 continue
142 if agent.vendor == "local":
143 warnings.append(
144 f"agent '{_redact_value(agent.name)}' (local) endpoint "
145 f"'{redact_url_userinfo(agent.endpoint or 'http://localhost:11434/v1')}' "
146 f"is not reachable"
147 )
148 elif agent.vendor in _HOSTED_API_VENDORS:
149 # Reuse the adapter's own capability warning (issue #430) instead
150 # of re-deriving the vendor -> env-var mapping here, so the
151 # message can't drift from what the adapter actually reports.
152 caps = _detect_capabilities(agent)
153 reason = "; ".join(caps.get("warnings", [])) or "the hosted API is not reachable"
154 warnings.append(f"agent '{_redact_value(agent.name)}' (hosted API): {reason}")
155 else:
156 warnings.append(
157 f"agent '{_redact_value(agent.name)}' command "
158 f"'{_redact_value(agent.command)}' is not on PATH"
159 )
160 return warnings
163def _recommendations(config_path, config_summary, agents) -> dict:
164 """Build actionable next-steps from the diagnostics (issue: doctor UX).
166 Returns ``{"ready": bool, "steps": [str, ...]}``. ``ready`` is true when at
167 least one agent is reachable. Steps point the user at the cheapest fix:
168 scaffold a config, install a CLI, or use a reachable local model.
169 """
170 steps: list[str] = []
171 available = [a for a in agents if a.get("available")]
172 ready = bool(available)
174 # No config file in play -> suggest scaffolding one.
175 if config_path is None and not Path("jury.toml").exists():
176 steps.append("No jury.toml found — run `jury init` to create one.")
178 if not ready:
179 from .adapters import list_local_models
181 models = list_local_models()
182 if models:
183 steps.append(
184 f"No agent CLI is available, but a local model server is reachable "
185 f"({len(models)} model(s): {', '.join(models[:3])}). Add a free local "
186 f"reviewer: `jury init --preset offline` (or `--list-models`)."
187 )
188 else:
189 steps.append(
190 "No reviewer is available. Install an agent CLI (claude / codex / agy), "
191 "or run a local model (e.g. `ollama serve` + `ollama pull "
192 'qwen2.5-coder:7b`) and add a `vendor = "local"` agent — or use '
193 "`--mock` for an offline demo."
194 )
195 else:
196 missing = [
197 a["name"]
198 for a in agents
199 if not a.get("available")
200 and config_summary
201 and a["name"] in config_summary.get("enabled_agents", [])
202 ]
203 if missing:
204 steps.append(
205 f"Enabled but unavailable (will be skipped): {', '.join(missing)}. "
206 f"Install them or run with `--strict` to fail instead."
207 )
209 return {"ready": ready, "steps": steps}
212def build_diagnostics(config_path=None):
213 """Build a SAFE diagnostics dict for the given config path.
215 Best-effort: if the config cannot be loaded, the error is captured as a
216 string under ``config_warnings`` and ``config`` is left ``None``. Never
217 raises for a bad/missing config. The returned dict never contains the raw
218 diff or any agent output.
219 """
220 config_summary = None
221 config_warnings: list[str] = []
222 agents: list = []
224 try:
225 cfg = load_config(config_path)
226 except FileNotFoundError as exc:
227 config_warnings.append(f"config error: {redact(str(exc))[0]}")
228 except tomllib.TOMLDecodeError as exc:
229 config_warnings.append(f"config error: invalid TOML: {redact(str(exc))[0]}")
230 except ConfigError as exc:
231 config_warnings.append(f"config error: {redact(str(exc))[0]}")
232 except (KeyError, ValueError, TypeError) as exc:
233 config_warnings.append(f"config error: {redact(str(exc))[0]}")
234 else:
235 config_summary = _config_summary(cfg)
236 agents = [_agent_entry(spec) for spec in cfg.agents]
237 config_warnings = _detect_warnings(cfg)
238 # Fold capability/version probe warnings (e.g. an available CLI whose
239 # version could not be detected) into the user-facing warnings list.
240 # Probes already ran while building the agent entries above.
241 enabled_names = {a.name for a in cfg.enabled_agents}
242 for spec, entry in zip(cfg.agents, agents, strict=False):
243 if spec.name not in enabled_names:
244 continue
245 for warning in entry.get("capability_warnings", []):
246 config_warnings.append(f"agent '{entry['name']}': {warning}")
248 return {
249 "tool_version": __version__,
250 "python_version": platform.python_version(),
251 "python_implementation": platform.python_implementation(),
252 "python_executable": sys.executable,
253 "os": platform.platform(),
254 "config_path": str(config_path) if config_path else "(default)",
255 "agents": agents,
256 "config": config_summary,
257 "config_warnings": config_warnings,
258 "recommendations": _recommendations(config_path, config_summary, agents),
259 }
262def render_report(diagnostics) -> str:
263 """Render a human-readable text report from a diagnostics dict."""
264 lines = []
265 lines.append("jury doctor")
266 lines.append("=" * 40)
267 lines.append(f"tool version: {diagnostics['tool_version']}")
268 lines.append(
269 f"python: {diagnostics['python_version']} ({diagnostics['python_implementation']})"
270 )
271 lines.append(f"python exe: {diagnostics['python_executable']}")
272 lines.append(f"os: {diagnostics['os']}")
273 lines.append(f"config path: {diagnostics['config_path']}")
274 lines.append("")
276 lines.append("Agents")
277 lines.append("-" * 40)
278 agents = diagnostics["agents"]
279 if not agents:
280 lines.append(" (no agents loaded)")
281 else:
282 for agent in agents:
283 status = "available" if agent["available"] else "MISSING"
284 lines.append(
285 f" [{status:>9}] {agent['name']} "
286 f"(vendor={agent['vendor']}, command={agent['command']})"
287 )
288 if agent.get("command") and agent.get("vendor") != "local": 288 ↛ 291line 288 didn't jump to line 291 because the condition on line 288 was always true
289 resolved = agent.get("resolved") or "(not found on PATH)"
290 lines.append(f" resolved: {resolved}")
291 version = agent.get("version") or "unknown"
292 caps = agent.get("capabilities") or {}
293 cap_bits = []
294 if caps.get("supports_headless"):
295 cap_bits.append("headless")
296 if caps.get("supports_model_selection"):
297 cap_bits.append("model-selection")
298 cap_summary = ", ".join(cap_bits) or "none"
299 cap_status = caps.get("status") or "unknown"
300 lines.append(
301 f" version={version}, capabilities=[{cap_summary}] "
302 f"(probe: {cap_status})"
303 )
304 lines.append("")
306 lines.append("Config summary")
307 lines.append("-" * 40)
308 config = diagnostics["config"]
309 if config is None:
310 lines.append(" (config could not be loaded)")
311 else:
312 lines.append(f" rounds: {config['rounds']}")
313 lines.append(f" chair: {config['chair']}")
314 lines.append(f" context mode: {config['context_mode']}")
315 enabled = ", ".join(config["enabled_agents"]) or "(none)"
316 lines.append(f" enabled: {enabled}")
317 lines.append("")
319 lines.append("Warnings")
320 lines.append("-" * 40)
321 warnings = diagnostics["config_warnings"]
322 if not warnings:
323 lines.append(" (none)")
324 else:
325 for warning in warnings:
326 lines.append(f" - {warning}")
327 lines.append("")
329 rec = diagnostics.get("recommendations") or {}
330 lines.append("Next steps")
331 lines.append("-" * 40)
332 lines.append(f" ready to run: {'yes' if rec.get('ready') else 'no'}")
333 for step in rec.get("steps", []):
334 lines.append(f" - {step}")
335 lines.append("")
337 lines.append(
338 "Privacy: no telemetry is collected or sent. This report is "
339 "local-only and redacts secret-like values."
340 )
342 return "\n".join(lines)