You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [6e344666] Improve sorterc scan speed, errors, and ingest compile. Make scan a fast DSL parse pass with human-readable output, surface full parse_error details, and add compile --ingest for single-event replay from a log. Co-authored-by: Cursor Side A — unified diff (full patch): diff --git a/server/src/offline.rs b/server/src/offline.rs index 54ad0ded096a305ef8454ab2cdd1c3af71b14f5d..2db6f67e22e00a24ce673d13095644dd9fa9342d 100644 --- a/server/src/offline.rs +++ b/server/src/offline.rs @@ -28,6 +28,10 @@ pub struct CompileResult { pub threads: Vec, pub rankings: Vec, pub stats: CompileStats, + #[serde(skip_serializing_if = "Option::is_none")] + pub ingest_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ingest_line: Option, } #[derive(Debug, Clone, Serialize)] @@ -36,6 +40,8 @@ pub struct CompileError { pub error: String, #[serde(skip_serializing_if = "Option::is_none")] pub hint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub parse_error: Option, } #[derive(Debug, Clone, Serialize)] @@ -50,7 +56,7 @@ pub struct MalformedIngest { pub id: String, pub room_id: String, pub thread_tag: String, - pub reason: String, + pub parse_error: String, } #[derive(Debug, Clone, Serialize)] @@ -59,9 +65,36 @@ pub struct ScanResult { pub path: String, pub total_lines: usize, pub parsed_events: usize, + pub ingest_events: usize, pub bad_json_lines: Vec, pub malformed_ingests: Vec, - pub skipped_ingests: usize, +} + +#[derive(Debug)] +pub enum CompileIngestError { + NotFound(String), + Io(std::io::Error), + Compile(CompileError), +} + +impl CompileIngestError { + pub fn into_compile_error(self) -> CompileError { + match self { + Self::NotFound(id) => CompileError { + ok: false, + error: format!("ingest not found: {id}"), + hint: Some("pass the ingest event id from events.jsonl".into()), + parse_error: None, + }, + Self::Io(e) => CompileError { + ok: false, + error: format!("io error: {e}"), + hint: None, + parse_error: None, + }, + Self::Compile(e) => e, + } + } } fn document_stats(doc: &dsl::Document) -> CompileStats { @@ -166,19 +199,22 @@ fn rankings_for_simulated( .collect() } -/// Validate and simulate one `.sorter` document against optional base reducer state. -pub fn compile_document( +fn compile_document_inner( base: &ReducerState, room: &str, text: &str, + ingest_id: Option, + ingest_line: Option, ) -> Result { let room_key = room.trim(); let scope = scope_from_room_wire(room_key); let validated = validate_ingest_document(base, text, &scope).map_err(|(_, message, hint)| { + let parse_error = dsl::parse_full(text).err().map(|e| e.to_string()); CompileError { ok: false, error: message, hint, + parse_error, } })?; @@ -200,15 +236,27 @@ pub fn compile_document( threads: threads_in_document(text), rankings: rankings_for_simulated(&simulated, &scope, room_key, &validated.doc), stats: document_stats(&validated.doc), + ingest_id, + ingest_line, }) } +/// Validate and simulate one `.sorter` document against optional base reducer state. +pub fn compile_document( + base: &ReducerState, + room: &str, + text: &str, +) -> Result { + compile_document_inner(base, room, text, None, None) +} + fn ingest_parse_error(raw: &str) -> Option { dsl::parse_full(raw).err().map(|e| e.to_string()) } -fn load_events_from_jsonl(path: &Path) -> Result<(Vec<(usize, Event)>, Vec), std::io::Error> { +fn load_events_from_jsonl(path: &Path) -> Result<(usize, Vec<(usize, Event)>, Vec), std::io::Error> { let text = std::fs::read_to_string(path)?; + let total_lines = text.lines().count(); let mut events = Vec::new(); let mut bad_json_lines = Vec::new(); for (idx, line) in text.lines().enumerate() { @@ -225,61 +273,90 @@ fn load_events_from_jsonl(path: &Path) -> Result<(Vec<(usize, Event)>, Vec Result<(ReducerState, Vec), std::io::Error> { - let (events, bad_json_lines) = load_events_from_jsonl(path)?; +fn replay_events(events: &[(usize, Event)]) -> ReducerState { let mut state = ReducerState::default(); for (_line_no, ev) in events { - state.apply_event(ev); + state.apply_event(ev.clone()); } - Ok((state, bad_json_lines)) + state } -/// Scan an events.jsonl for corrupt JSON lines and ingests that fail DSL replay. -pub fn scan_jsonl(path: &Path) -> Result { - let text = std::fs::read_to_string(path)?; - let total_lines = text.lines().count(); - let (events, bad_json_lines) = load_events_from_jsonl(path)?; +/// Replay a JSONL event log into reducer state (same rules as server boot). +pub fn load_reducer_from_jsonl(path: &Path) -> Result<(ReducerState, Vec), std::io::Error> { + let (_total_lines, events, bad_json_lines) = load_events_from_jsonl(path)?; + Ok((replay_events(&events), bad_json_lines)) +} - let mut malformed_ingests = Vec::new(); - let mut skipped_ingests = 0usize; - let mut state = ReducerState::default(); - let parsed_events = events.len(); +/// Find one ingest in a log and compile it against all prior events as base state. +pub fn compile_ingest_from_log(path: &Path, ingest_id: &str) -> Result { + let (_total_lines, events, bad_json_lines) = load_events_from_jsonl(path).map_err(CompileIngestError::Io)?; + if !bad_json_lines.is_empty() { + return Err(CompileIngestError::Compile(CompileError { + ok: false, + error: format!("jsonl has {} corrupt line(s)", bad_json_lines.len()), + hint: Some("fix the log or use `sorterc scan`".into()), + parse_error: None, + })); + } + + let needle = ingest_id.trim(); + let mut found: Option<(usize, Ingest)> = None; + let mut prior: Vec<(usize, Event)> = Vec::new(); for (line_no, ev) in events { if let Event::Ingest(ref ing) = ev { - if let Some(reason) = ingest_parse_error(&ing.raw) { + if ing.id == needle { + found = Some((line_no, ing.clone())); + break; + } + } + prior.push((line_no, ev)); + } + + let (line_no, ing) = found.ok_or_else(|| CompileIngestError::NotFound(needle.to_string()))?; + let base = replay_events(&prior); + compile_document_inner(&base, &ing.room_id, &ing.raw, Some(ing.id.clone()), Some(line_no)) + .map_err(CompileIngestError::Compile) +} + +/// Scan an events.jsonl for corrupt JSON lines and ingests whose DSL fails to parse. +/// +/// This does not replay the log (which would run rank centrality on every ingest and +/// can take minutes on real logs). It matches what the server skips on boot: parse failure. +pub fn scan_jsonl(path: &Path) -> Result { + let (total_lines, events, bad_json_lines) = load_events_from_jsonl(path)?; + + let mut malformed_ingests = Vec::new(); + let mut ingest_events = 0usize; + + for (line_no, ev) in &events { + if let Event::Ingest(ing) = ev { + ingest_events += 1; + if let Some(parse_error) = ingest_parse_error(&ing.raw) { malformed_ingests.push(MalformedIngest { - line: line_no, + line: *line_no, id: ing.id.clone(), room_id: ing.room_id.clone(), thread_tag: ing.thread_tag.clone(), - reason, + parse_error, }); } - let before = state.ingests_by_id.len(); - state.apply_event(ev); - if state.ingests_by_id.len() == before { - skipped_ingests += 1; - } - } else { - state.apply_event(ev); } } - let ok = bad_json_lines.is_empty() && malformed_ingests.is_empty() && skipped_ingests == 0; + let ok = bad_json_lines.is_empty() && malformed_ingests.is_empty(); Ok(ScanResult { ok, path: path.display().to_string(), total_lines, - parsed_events, + parsed_events: events.len(), + ingest_events, bad_json_lines, malformed_ingests, - skipped_ingests, }) } @@ -330,4 +407,25 @@ mod tests { assert!(!report.ok); assert_eq!(report.bad_json_lines.len(), 1); } + + #[test] + fn scan_reports_dsl_parse_error_detail() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("events.jsonl"); + let ingest = serde_json::json!({ + "type": "ingest", + "ts": 1, + "id": "bad-ingest-id", + "raw": "{ no closing brace\n~/a { body }\n~/a 1:0 ~/b", + "principal": "test", + "room_id": "public", + "thread_tag": "t", + }); + std::fs::write(&path, format!("{ingest}\n")).unwrap(); + let report = scan_jsonl(&path).unwrap(); + assert!(!report.ok); + assert_eq!(report.malformed_ingests.len(), 1); + assert_eq!(report.malformed_ingests[0].id, "bad-ingest-id"); + assert!(report.malformed_ingests[0].parse_error.contains("parse error")); + } } diff --git a/sorterc/readme.md b/sorterc/readme.md index 1ebcc3fc935541ea9e47e0458ec67a750fe19fff..e388615c34b40914ca51fc7a79cb738eebc2909b 100644 --- a/sorterc/readme.md +++ b/sorterc/readme.md @@ -60,21 +60,33 @@ Rankings use the same structure as the server's dry-run check: parent scope, con ### `scan` — lint an `events.jsonl` -Reads a JSONL event log and reports problems without starting a server. +Fast single-pass check. Does **not** replay the log (replay runs rank centrality on every ingest and gets slow fast). ```bash cargo run -p sorterc -- scan events.jsonl -cargo run -p sorterc -- scan events.jsonl --pretty +cargo run -p sorterc -- scan events.jsonl --json +cargo run -p sorterc -- scan events.jsonl --json --pretty ``` +Default output is human-readable with a blank line between each problem. Use `--json` for machine output. + Reports: - **bad JSON lines** — lines that are not valid JSON -- **malformed ingests** — ingest events whose `raw` DSL fails to parse -- **skipped ingests** — ingests dropped during replay (same behavior as server boot) +- **malformed ingests** — ingest events whose `raw` DSL fails to parse, with full `parse_error` text Exits 0 when clean, 1 when any issue is found. +### `compile --ingest` — compile one event from a log + +Replay all events **before** the target ingest as base state, then compile that ingest's DSL: + +```bash +cargo run -p sorterc -- compile --ingest cabd8adc-57ae-402d-a940-8e24339ac451 --from events.jsonl --pretty +``` + +Output includes `ingest_id`, `ingest_line`, and rankings for that post only. This may take a while for ingests late in a large log (full replay up to that point). + ## Typical uses - Iterate on `.sorter` files in an editor and pipe through `compile` to see rankings instantly diff --git a/sorterc/src/main.rs b/sorterc/src/main.rs index 71382c180085cb0ad71043c852f8db5d3a48a284..5687c850d16973d28749380b8dc89f3bcebbfc56 100644 --- a/sorterc/src/main.rs +++ b/sorterc/src/main.rs @@ -22,9 +22,15 @@ struct Cli { enum Command { /// Parse and simulate a .sorter document; emit ranking JSON to stdout. Compile { - /// `.sorter` file, or `-` for stdin. - file: PathBuf, - /// Room wire id (`public` or private room id). + /// `.sorter` file, or `-` for stdin. Omit when using --ingest. + file: Option, + /// Compile one ingest event from a log (by event id / uuid). + #[arg(long)] + ingest: Option, + /// events.jsonl containing the ingest (required with --ingest). + #[arg(long)] + from: Option, + /// Room wire id (`public` or private room id). Ignored with --ingest. #[arg(long, default_value = "public")] room: String, /// Optional events.jsonl to replay before compiling (seed garden state). @@ -34,9 +40,13 @@ enum Command { #[arg(long)] pretty: bool, }, - /// Scan an events.jsonl for corrupt JSON lines and malformed ingests. + /// Scan an events.jsonl for corrupt JSON lines and DSL parse failures. Scan { file: PathBuf, + /// Emit JSON instead of human-readable output. + #[arg(long)] + json: bool, + /// Pretty-print JSON (requires --json). #[arg(long)] pretty: bool, }, @@ -58,6 +68,7 @@ fn load_base_state(base: Option<&Path>) -> Result { let Some(path) = base else { return Ok(ReducerState::default()); }; + eprintln!("sorterc: replaying {} for base state (may take a while on large logs)…", path.display()); let (state, bad_lines) = offline::load_reducer_from_jsonl(path) .with_context(|| format!("load base jsonl {}", path.display()))?; if !bad_lines.is_empty() { @@ -78,25 +89,101 @@ fn print_json(value: &T, pretty: bool) -> Result<()> { Ok(()) } -fn run_compile(file: PathBuf, room: String, base: Option, pretty: bool) -> Result<()> { - let text = read_input(&file)?; - let base_state = load_base_state(base.as_deref())?; - match offline::compile_document(&base_state, &room, &text) { - Ok(result) => { - print_json::(&result, pretty)?; - Ok(()) +fn run_compile( + file: Option, + ingest: Option, + from: Option, + room: String, + base: Option, + pretty: bool, +) -> Result<()> { + if ingest.is_some() ^ from.is_some() { + bail!("--ingest and --from must be used together"); + } + if ingest.is_some() && (file.is_some() || base.is_some()) { + bail!("with --ingest/--from, omit file and --base"); + } + if file.is_none() && ingest.is_none() { + bail!("pass a .sorter file or --ingest --from events.jsonl"); + } + + if let (Some(ingest_id), Some(log_path)) = (ingest, from) { + eprintln!( + "sorterc: compiling ingest {ingest_id} from {}…", + log_path.display() + ); + match offline::compile_ingest_from_log(&log_path, &ingest_id) { + Ok(result) => { + print_json::(&result, pretty)?; + Ok(()) + } + Err(err) => { + print_json::(&err.into_compile_error(), pretty)?; + std::process::exit(1); + } + } + } else { + let file = file.expect("checked above"); + let text = read_input(&file)?; + let base_state = load_base_state(base.as_deref())?; + match offline::compile_document(&base_state, &room, &text) { + Ok(result) => { + print_json::(&result, pretty)?; + Ok(()) + } + Err(err) => { + print_json::(&err, pretty)?; + std::process::exit(1); + } + } + } +} + +fn print_scan_human(report: &ScanResult) { + if report.ok { + println!( + "ok: {} ({} lines, {} events, {} ingests)", + report.path, report.total_lines, report.parsed_events, report.ingest_events + ); + return; + } + + let problems = report.bad_json_lines.len() + report.malformed_ingests.len(); + println!( + "{}: {} lines, {} events, {} ingests, {problems} problem(s)", + report.path, report.total_lines, report.parsed_events, report.ingest_events + ); + + let mut first = true; + for bad in &report.bad_json_lines { + if !first { + println!(); } - Err(err) => { - print_json::(&err, pretty)?; - std::process::exit(1); + first = false; + println!("line {}: invalid JSON", bad.line); + println!("{}", bad.message); + } + + for bad in &report.malformed_ingests { + if !first { + println!(); } + first = false; + println!( + "line {}: ingest {} (#{} in {})", + bad.line, bad.id, bad.thread_tag, bad.room_id + ); + println!("{}", bad.parse_error); } } -fn run_scan(file: PathBuf, pretty: bool) -> Result<()> { - let report = offline::scan_jsonl(&file) - .with_context(|| format!("scan {}", file.display()))?; - print_json::(&report, pretty)?; +fn run_scan(file: PathBuf, json: bool, pretty: bool) -> Result<()> { + let report = offline::scan_jsonl(&file).with_context(|| format!("scan {}", file.display()))?; + if json { + print_json::(&report, pretty)?; + } else { + print_scan_human(&report); + } if !report.ok { std::process::exit(1); } @@ -108,10 +195,12 @@ fn main() -> Result<()> { match cli.cmd { Command::Compile { file, + ingest, + from, room, base, pretty, - } => run_compile(file, room, base, pretty), - Command::Scan { file, pretty } => run_scan(file, pretty), + } => run_compile(file, ingest, from, room, base, pretty), + Command::Scan { file, json, pretty } => run_scan(file, json, pretty), } } Side B — contributor: tommy-mor Side B — commit message: [71f27882] Delete test/runner.clj Side B — unified diff (full patch): diff --git a/test/runner.clj b/test/runner.clj deleted file mode 100644 index e8d3d3d05e84110545d68350b7db2c9d1cfbf979..0000000000000000000000000000000000000000 --- a/test/runner.clj +++ /dev/null @@ -1,24 +0,0 @@ -(ns test.runner - "JVM entrypoint: run the same flows as Kaocha (for scripts / quick runs)." - (:require [test.integration :as integration] - [test.auth :as auth] - [test.grants :as grants] - [test.invites :as invites] - [test.room-list :as room-list] - [test.browser-sse :as browser-sse] - [test.browser-post-redact :as browser-post-redact] - [test.browser-ui-morph :as browser-ui-morph])) - -(defn run-core! [] - (integration/integration-flow!) - (auth/auth-flow!) - (grants/grants-flow!) - (invites/invites-flow!) - (room-list/room-list-flow!)) - -(defn -main [& args] - (case (vec args) - ["browser-sse"] (browser-sse/sse-browser-flow!) - ["browser-post-redact"] (browser-post-redact/post-redact-flow!) - ["browser-ui-morph"] (browser-ui-morph/ui-morph-flow!) - (run-core!)))