diff --git a/constitution.py b/constitution.py index 37239ba74c1ba9460be009dd8f4a402aa9c32fe0..2e5a05a59700c59da7e08a11620c8efe48b99f32 100644 --- a/constitution.py +++ b/constitution.py @@ -503,9 +503,18 @@ def _epochs_in_ledger() -> list[int]: def build_pairwise_prompt(side_a: dict, side_b: dict) -> str: - return f"""You are ranking individual git commits to an open source project. -Compare these two commits. Decide which commit contributed more. + return f"""You are a constitutional council ranking individual git commits for ownership allocation. + +Compare these two commits. Decide which contributed more lasting value to the project. + +Judge substance, not spectacle: +- Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. +- Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. +- Do not favor a side merely because its patch is longer or noisier. +- Weight what the change does for the project, not the contributor's name. + Return ONLY a JSON object: {{"winner": "A" or "B", "ratio": "N:M", "explanation": "..."}} +The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: {side_a.get('contributor', '?')} Side A — commit message: @@ -801,9 +810,14 @@ async def _fetch_models_from_slug_rank_parent( return out +def _model_provider(model_id: str) -> str: + return (model_id or "").split("/", 1)[0] or model_id + + async def _fetch_models_openrouter_only( client: httpx.AsyncClient, n: int, exclude: set[str] ) -> list[str]: + """Top up council seats, preferring provider diversity over same-lab clones.""" key = (OPENROUTER_API_KEY or "").strip() if not key or n <= 0: return [] @@ -815,10 +829,24 @@ async def _fetch_models_openrouter_only( models = resp.json()["data"] chat_models = [m for m in models if "chat" in m.get("id", "")] chat_models.sort(key=lambda m: m.get("created", 0), reverse=True) - out = [] + out: list[str] = [] + used_providers = {_model_provider(m) for m in exclude} + # Pass 1: one seat per unused provider. for m in chat_models: mid = m["id"] - if mid in exclude: + if mid in exclude or mid in out: + continue + provider = _model_provider(mid) + if provider in used_providers: + continue + out.append(mid) + used_providers.add(provider) + if len(out) >= n: + return out + # Pass 2: fill remaining seats with newest chat models. + for m in chat_models: + mid = m["id"] + if mid in exclude or mid in out: continue out.append(mid) if len(out) >= n: @@ -1639,6 +1667,10 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): sides = [_commit_side_for_llm(row) for row in ordered] judgment_ids: list[str] = [] + # A dead juror must not halt the court. Retire after hard failure; continue + # with remaining models. Abort only when a comparison gets zero votes. + retired_models: set[str] = set() + models_that_voted: set[str] = set() async def compare_fn(i, j): side_a, side_b = sides[i], sides[j] @@ -1690,6 +1722,8 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): ])) results = [] for model in models: + if model in retired_models: + continue try: existing = _find_judgment(comparison_id, model) if existing: @@ -1715,6 +1749,7 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): ratio = result["ratio"].split(":") winner_weight, loser_weight = float(ratio[0]), float(ratio[1]) results.append((w, l, winner_weight, loser_weight)) + models_that_voted.add(model) jud_id = (existing or _find_judgment(comparison_id, model) or {}).get( "judgment_id" ) @@ -1747,15 +1782,31 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): ] ])) except Exception as e: + retired_models.add(model) + await append_evidence(epoch, "llm.council_member_failed", { + "ranking_run_id": ranking_run_id, + "comparison_id": comparison_id, + "model_id": model, + "error": {"type": type(e).__name__, "message": str(e)}, + "summary": ( + f"Retired {model} from this ranking run after hard failure" + ), + }) await broadcast_audit( "error", - f"{model} failed: {e}", - phase="error", + f"{model} failed and was retired from this run: {e}", + phase="ranking", + evidence_url=_evidence_url("comparison", comparison_id), + links={"comparison": _evidence_url("comparison", comparison_id)}, ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ - ["div.log-error", f"⚠ {model}: {e}"] + ["div.log-error", + f"⚠ {model} retired after failure: {e}"] ])) - raise RuntimeError(f"council model failed: {model}") from e + if not results: + raise RuntimeError( + f"council model failed: no votes for comparison {comparison_id}" + ) return results async def progress_fn(ev): @@ -1793,9 +1844,12 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): key=lambda x: x[1], reverse=True, ) + models_voted = sorted(models_that_voted) or list(models) completed = await append_evidence(epoch, "ranking.completed", { "ranking_run_id": ranking_run_id, "models": models, + "models_voted": models_voted, + "models_failed": sorted(retired_models), "commit_ranking": commit_ranking, "contributor_ranking": {a: str(s) for a, s in contrib_rows}, "ranking": {a: str(s) for a, s in contrib_rows}, @@ -1806,6 +1860,10 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): f"{cid[:16]}={float(s):.4f}" for cid, s, _ in commit_rows[:12] ) + ("…" if len(commit_rows) > 12 else "") + + ( + f" (retired: {', '.join(sorted(retired_models))})" + if retired_models else "" + ) ), }) await broadcast_audit( @@ -1824,7 +1882,7 @@ async def rank_commits(commits: list[dict], *, epoch: int = -1): *[["span.rank-entry", f"{a} {float(s):.3f} "] for a, s in contrib_rows], ] ])) - return contributor_totals, models, { + return contributor_totals, models_voted, { "ranking_run_id": ranking_run_id, "ranking_event_id": completed.event_id, } @@ -2413,6 +2471,7 @@ async def epoch_detail(epoch: int): # Dense comparison cards with inline reasoning (permalinks kept). comparison_cards: list = [] + disagreement_cards: list = [] for cmp in comparison_evs: cid = cmp.payload.get("comparison_id") if not cid: @@ -2421,8 +2480,11 @@ async def epoch_detail(epoch: int): side_b = cmp.payload.get("side_b") or {} juds = _judgments_for_comparison(cid) reason_lines = [] + winners: set[str] = set() for j in juds: p = j.payload + if p.get("winner"): + winners.add(str(p["winner"])) expl = (p.get("explanation") or "").strip() if len(expl) > 220: expl = expl[:220] + "…" @@ -2431,16 +2493,21 @@ async def epoch_detail(epoch: int): f" → {p.get('winner')} ({p.get('ratio')}) — {expl} ", _a(_evidence_path("judgment", p.get("judgment_id") or j.event_id), "↗"), ]) - comparison_cards.append(["article.dense-card", + card = ["article.dense-card", ["div.card-head", _a(_evidence_path("comparison", cid), "comparison"), " · ", _side_label(side_a), " vs ", _side_label(side_b), + *([" · ", ["span.disagree", "disagreement"]] + if len(winners) > 1 else []), ], ["ul.reason-list", *reason_lines] if reason_lines else ["p.note", "No judgments yet."], - ]) + ] + comparison_cards.append(card) + if len(winners) > 1: + disagreement_cards.append(card) ranking_nodes: list = [] if ranking_completed: @@ -2468,9 +2535,14 @@ async def epoch_detail(epoch: int): ["td", who or "?"], ["td.msg", msg], ]) + models_failed = ranking_completed.payload.get("models_failed") or [] ranking_nodes = [ ["p.note", ranking_completed.payload.get("summary") or "ranking completed"], ["p", "Models: ", ", ".join(ranking_completed.payload.get("models") or []) or "—"], + *( + [["p.note", "Retired mid-run: ", ", ".join(models_failed)]] + if models_failed else [] + ), ["h3", "commit ranking"], ["table.dense", ["thead", ["tr", @@ -2559,6 +2631,12 @@ async def epoch_detail(epoch: int): emission_node, ["h2", "ranking"], *ranking_nodes, + ["h2", "council disagreements"], + *( + disagreement_cards + if disagreement_cards + else [["p.note", "No split votes yet — council unanimous where judged."]] + ), ["h2", "comparisons"], *(comparison_cards if comparison_cards else [["p.note", no_comparisons_note]]), _fold("All commits", _link_list(commit_links)), @@ -3147,6 +3225,7 @@ p.lede { color: var(--ui); margin: 0 0 1em; } .card-head { color: var(--ui); font-family: var(--font-code); font-size: 12px; margin-bottom: 6px; } .reason-list { margin: 0; padding-left: 1.1em; } .reason-list li { color: var(--prose); font-family: var(--font-prose); line-height: 1.45; } +.disagree { color: #c9a227; font-family: var(--font-code); font-size: 12px; } .judgment { background: var(--g1); border-left: 3px solid var(--link); diff --git a/tests/test_evidence.py b/tests/test_evidence.py index 41b7f03a2cf3a6d684d60679a5081053836f98c2..fd354fa4ed5c6af212a71476d6484d2a4561587e 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -92,6 +92,40 @@ def test_commit_without_evidence_is_not_found(evidence_store): assert "legacy" not in html.lower() +def test_pairwise_prompt_rejects_patch_theater(): + prompt = c.build_pairwise_prompt( + {"contributor": "a", "message": "m", "diff": "d"}, + {"contributor": "b", "message": "m", "diff": "d"}, + ) + assert "lasting value" in prompt + assert "line count" in prompt + + +def test_epoch_page_surfaces_council_disagreements(evidence_store): + cmp_id = "cmp_disagree" + asyncio.run(c.append_evidence(2, "comparison.input", { + "comparison_id": cmp_id, + "summary": "A vs B", + "side_a": {"commit_id": "c_a", "contributor": "alice"}, + "side_b": {"commit_id": "c_b", "contributor": "bob"}, + "prompt": c._bytes_blob("p"), + })) + for model, winner in (("mock/a", "A"), ("mock/b", "B")): + asyncio.run(c.append_evidence(2, "llm.judgment", { + "judgment_id": f"jud_{model}", + "comparison_id": cmp_id, + "model_id": model, + "winner": winner, + "ratio": "2:1", + "explanation": f"{model} picked {winner}", + "summary": f"{model}: {winner}", + })) + html = asyncio.run(c.epoch_detail(2)).body.decode() + assert "council disagreements" in html + assert "disagreement" in html + assert f"/comparisons/{cmp_id}" in html + + def test_comparison_page_shows_reasoning_inline(evidence_store): cmp_id = "cmp_test_dense" jud_id = "jud_test_dense" diff --git a/tests/test_git_discovery.py b/tests/test_git_discovery.py index 0f002a58bd9a122d41c5a85e32c16ed2b4404d2f..b50426a4ca35b7678f29d778306aced11b4954f1 100644 --- a/tests/test_git_discovery.py +++ b/tests/test_git_discovery.py @@ -513,7 +513,7 @@ def test_same_contributor_multiple_commits_runs_pairwise( assert info["ranking_event_id"] -def test_any_council_failure_aborts_ranking(discovery_config, monkeypatch): +def test_all_council_failures_abort_ranking(discovery_config, monkeypatch): monkeypatch.setattr(c, "store", c.JsonlStore(discovery_config / "ledger.jsonl")) async def models(n=3): @@ -538,6 +538,64 @@ def test_any_council_failure_aborts_ranking(discovery_config, monkeypatch): asyncio.run(c.rank_commits(commits, epoch=0)) +def test_one_council_failure_retires_model_and_continues(discovery_config, monkeypatch): + monkeypatch.setattr(c, "store", c.JsonlStore(discovery_config / "ledger.jsonl")) + + async def models(n=3): + return ["broken", "solid"] + + async def compare(model_id, side_a, side_b, **kwargs): + if model_id == "broken": + raise RuntimeError("model unavailable") + if kwargs.get("persist"): + attempt_id = c.attempt_id_for(kwargs["comparison_id"], model_id, 1) + await c.append_evidence(kwargs["epoch"], "llm.judgment", { + "judgment_id": c.judgment_id_for({ + "attempt_id": attempt_id, + "comparison_id": kwargs["comparison_id"], + "model_id": model_id, + "winner": "A", + "ratio": "2:1", + "explanation": "ok", + }), + "attempt_id": attempt_id, + "comparison_id": kwargs["comparison_id"], + "model_id": model_id, + "winner": "A", + "ratio": "2:1", + "explanation": "ok", + "summary": "ok", + }) + return {"winner": "A", "ratio": "2:1", "explanation": "ok"} + + monkeypatch.setattr(c, "fetch_top_models", models) + monkeypatch.setattr(c, "llm_pairwise_compare", compare) + monkeypatch.setattr(c, "OPENROUTER_API_KEY", "test-key") + commits = [ + { + "contributor": contributor, + "oid": "sha1:" + char * 40, + "message": contributor, + "patch": "patch", + } + for contributor, char in [("alice", "a"), ("bob", "b")] + ] + ranking, used, _info = asyncio.run(c.rank_commits(commits, epoch=0)) + assert set(ranking) == {"alice", "bob"} + assert used == ["solid"] + retired = [ + e for e in c.store.read() + if isinstance(e, c.Evidence) and e.kind == "llm.council_member_failed" + ] + assert len(retired) == 1 + assert retired[0].payload["model_id"] == "broken" + completed = next( + e for e in c.store.read() + if isinstance(e, c.Evidence) and e.kind == "ranking.completed" + ) + assert completed.payload["models_failed"] == ["broken"] + + def test_multi_commit_ranking_requires_openrouter_key(monkeypatch): monkeypatch.setattr(c, "OPENROUTER_API_KEY", "") commits = [