You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [239c074b] url schema stuff Side A — unified diff (full patch): diff --git a/AGENTS.md b/AGENTS.md index 426a88e7c1da54fe0a28c5c76fa4e1f1bc117fcf..e60b9ba6012593361ef10e8fdd9439cd9932e09b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,3 +58,4 @@ Use **tmux** for `cargo run --package sorter2-server` (dev server). Rebuild afte - First `cargo test` / `cargo build --release` is slow; Clojure smoke test always does a release build. - `legacy/` and `ideas/` are not part of the workspace build. +- **ItemId** for web URLs is a canonical full URL (`https://reddit.com/r/rust`). Rules live in [`server/src/url_rules/`](server/src/url_rules/) (composable Rust, not a config DSL). After changing canonicalization rules, rebuild the projection: `cargo run --package sorter2-server -- replay-index`. diff --git a/Cargo.lock b/Cargo.lock index 0dd4fce5fb6400ae153cca4e3dbf5a5158e6d8b4..49a908ef935c430dbe63c6a28d8a24e38b489486 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1951,6 +1951,7 @@ dependencies = [ "tower-http 0.5.2", "tracing", "tracing-subscriber", + "url", "urlencoding", ] diff --git a/REPLAY.sh b/REPLAY.sh new file mode 100755 index 0000000000000000000000000000000000000000..f2dbd8aea60c02d2feef74805f7ef5c2b7022537 --- /dev/null +++ b/REPLAY.sh @@ -0,0 +1,2 @@ +cargo run --package sorter2-server -- replay-index + diff --git a/server/Cargo.toml b/server/Cargo.toml index 27f552c20b97ef28cdde4cb6b1a4980375135111..ad4912791aff59fb1d3293f66ad381ae618cd60b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -24,6 +24,7 @@ async-stream = "0.3" futures-util = { version = "0.3", default-features = false, features = ["std"] } rand = "0.8" urlencoding = "2" +url = "2" durable = { path = "../durable" } [dev-dependencies] diff --git a/server/src/entity_store.rs b/server/src/entity_store.rs index d5d17c3676e4a8ddec998e9f5a9dbafe9c2d9d0e..d29f39aecca6f12cdcf263cf77c3654eb4ee6cfa 100644 --- a/server/src/entity_store.rs +++ b/server/src/entity_store.rs @@ -124,7 +124,7 @@ mod tests { fn round_trip_payload() { let tmp = tempfile::tempdir().unwrap(); let store = EntityStore::open(tmp.path()).unwrap(); - let id = ItemId::parse("reddit.com/r/rust").unwrap(); + let id = ItemId::from_url("https://reddit.com/r/rust").unwrap(); let payload = json!({"kind": "t5", "data": {"display_name": "rust"}}); store.put(&id, &payload).unwrap(); diff --git a/server/src/event_log.rs b/server/src/event_log.rs index 36f5b406084065b608735987cdb483c236e03081..2c9290b6fdbf2c2ad1c0f1ffd7374b2d9cc97f36 100644 --- a/server/src/event_log.rs +++ b/server/src/event_log.rs @@ -199,7 +199,7 @@ mod tests { log.append(&sample_record( 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )) .await @@ -237,7 +237,7 @@ mod tests { let path = tmp.path().join("events.jsonl"); let log = EventLog::new(&path); let event = Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }; log.append(&sample_record(1, event)).await.unwrap(); @@ -255,7 +255,7 @@ mod tests { let path = tmp.path().join("events.jsonl"); std::fs::write( &path, - r#"{"type":"node_ensured","id":"reddit.com/r/rust"} + r#"{"type":"node_ensured","id":"https://reddit.com/r/rust"} {"schema":1,"seq":1,"ts":1,"event":{"type":"vote_recorded","ts":1,"a":"a","b":"b","ratio_left":2,"ratio_right":1,"scope":""}} "#, ) @@ -295,7 +295,7 @@ mod tests { log.append(&sample_record( 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )) .await @@ -303,7 +303,7 @@ mod tests { log.append(&sample_record( 3, Event::NodeEnsured { - id: "reddit.com/r/python".into(), + id: "https://reddit.com/r/python".into(), }, )) .await diff --git a/server/src/journal.rs b/server/src/journal.rs index 521a108019de1ea870d14c4fafbfe572c20ce0de..50bc89f976edb82b7b0e49e954a8eccbbe82bf87 100644 --- a/server/src/journal.rs +++ b/server/src/journal.rs @@ -141,10 +141,10 @@ mod tests { let j2 = journal.clone(); let (r1, r2) = tokio::join!( j1.append(Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }), j2.append(Event::NodeEnsured { - id: "reddit.com/r/python".into(), + id: "https://reddit.com/r/python".into(), }), ); r1.unwrap(); @@ -153,10 +153,10 @@ mod tests { assert_eq!(projection_store.last_applied_event_count().unwrap(), 2); let tree = projection_store.load_tree().unwrap(); assert!(tree - .get(&ItemId::parse("reddit.com/r/rust").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .is_some()); assert!(tree - .get(&ItemId::parse("reddit.com/r/python").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/python").unwrap()) .is_some()); } @@ -170,7 +170,7 @@ mod tests { 1, 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )) .await @@ -186,7 +186,7 @@ mod tests { 1, 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )], ) @@ -202,7 +202,7 @@ mod tests { ); journal .append(Event::NodeEnsured { - id: "reddit.com/r/python".into(), + id: "https://reddit.com/r/python".into(), }) .await .unwrap(); @@ -227,13 +227,13 @@ mod tests { journal .append_many(vec![ Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, Event::NodeEnsured { - id: "reddit.com/r/python".into(), + id: "https://reddit.com/r/python".into(), }, Event::NodeEnsured { - id: "reddit.com/r/clojure".into(), + id: "https://reddit.com/r/clojure".into(), }, ]) .await @@ -245,7 +245,7 @@ mod tests { assert_eq!(projection_store.last_applied_event_count().unwrap(), 3); let tree = projection_store.load_tree().unwrap(); assert!(tree - .get(&ItemId::parse("reddit.com/r/clojure").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/clojure").unwrap()) .is_some()); } } diff --git a/server/src/lib.rs b/server/src/lib.rs index 9bd5f76fd1406b9b1be4c272f4ba8647edde2678..5c02c8e704e4664453bad75d819df8a067668176 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -9,6 +9,7 @@ pub mod journal; pub mod pair; pub mod parser; pub mod path_types; +pub mod url_rules; pub mod projection_apply; pub mod projection_store; pub mod ranking; diff --git a/server/src/pair.rs b/server/src/pair.rs index 43f780ba6ea6ce1cdc2e1f4cbb252ba8a10684b9..815a97b80e3e9f348e0937a4f147f2862018edb0 100644 --- a/server/src/pair.rs +++ b/server/src/pair.rs @@ -381,42 +381,42 @@ mod tests { #[test] fn suggest_prefers_unvoted_pair() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); let mut tree = seed_children( &parent, &[ - "reddit.com/r/rust/a", - "reddit.com/r/rust/b", - "reddit.com/r/rust/c", + "https://reddit.com/r/rust/a", + "https://reddit.com/r/rust/b", + "https://reddit.com/r/rust/c", ], ); let vote = - VoteData::from_recorded(1, "reddit.com/r/rust/a", "reddit.com/r/rust/b", 2, 1).unwrap(); + VoteData::from_recorded(1, "https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b", 2, 1).unwrap(); tree.apply_vote(&parent, vote); let group = tree.get(&parent).unwrap().local_ranking.clone(); let pool = children_of(&tree, &parent); let (l, r) = suggest_next_pair_in_pool(&group, &pool, None).unwrap(); - let voted_ab = (l.as_str() == "reddit.com/r/rust/a" && r.as_str() == "reddit.com/r/rust/b") - || (l.as_str() == "reddit.com/r/rust/b" && r.as_str() == "reddit.com/r/rust/a"); + let voted_ab = (l.as_str() == "https://reddit.com/r/rust/a" && r.as_str() == "https://reddit.com/r/rust/b") + || (l.as_str() == "https://reddit.com/r/rust/b" && r.as_str() == "https://reddit.com/r/rust/a"); assert!(!voted_ab); } #[test] fn suggest_bridges_separate_components() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); let mut tree = seed_children( &parent, &[ - "reddit.com/r/rust/a", - "reddit.com/r/rust/b", - "reddit.com/r/rust/c", - "reddit.com/r/rust/d", + "https://reddit.com/r/rust/a", + "https://reddit.com/r/rust/b", + "https://reddit.com/r/rust/c", + "https://reddit.com/r/rust/d", ], ); let ab = - VoteData::from_recorded(1, "reddit.com/r/rust/a", "reddit.com/r/rust/b", 2, 1).unwrap(); + VoteData::from_recorded(1, "https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b", 2, 1).unwrap(); let cd = - VoteData::from_recorded(2, "reddit.com/r/rust/c", "reddit.com/r/rust/d", 2, 1).unwrap(); + VoteData::from_recorded(2, "https://reddit.com/r/rust/c", "https://reddit.com/r/rust/d", 2, 1).unwrap(); tree.apply_vote(&parent, ab); tree.apply_vote(&parent, cd); let group = tree.get(&parent).unwrap().local_ranking.clone(); @@ -424,37 +424,37 @@ mod tests { let pair = suggest_next_pair_in_pool(&group, &pool, None).unwrap(); let chosen = pair_set(&pair); let from_ab = - chosen.contains("reddit.com/r/rust/a") || chosen.contains("reddit.com/r/rust/b"); + chosen.contains("https://reddit.com/r/rust/a") || chosen.contains("https://reddit.com/r/rust/b"); let from_cd = - chosen.contains("reddit.com/r/rust/c") || chosen.contains("reddit.com/r/rust/d"); + chosen.contains("https://reddit.com/r/rust/c") || chosen.contains("https://reddit.com/r/rust/d"); assert!(from_ab && from_cd, "expected bridge pair, got {:?}", chosen); } #[test] fn suggest_prefers_attach_over_isolate_pair_among_many_unranked() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); let mut tree = seed_children( &parent, &[ - "reddit.com/r/rust/a", - "reddit.com/r/rust/b", - "reddit.com/r/rust/c", - "reddit.com/r/rust/d", - "reddit.com/r/rust/e", + "https://reddit.com/r/rust/a", + "https://reddit.com/r/rust/b", + "https://reddit.com/r/rust/c", + "https://reddit.com/r/rust/d", + "https://reddit.com/r/rust/e", ], ); let ab = - VoteData::from_recorded(1, "reddit.com/r/rust/a", "reddit.com/r/rust/b", 2, 1).unwrap(); + VoteData::from_recorded(1, "https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b", 2, 1).unwrap(); tree.apply_vote(&parent, ab); let group = tree.get(&parent).unwrap().local_ranking.clone(); let pool = children_of(&tree, &parent); let pair = suggest_next_pair_in_pool(&group, &pool, None).unwrap(); let chosen = pair_set(&pair); let from_ab = - chosen.contains("reddit.com/r/rust/a") || chosen.contains("reddit.com/r/rust/b"); - let from_cde = chosen.contains("reddit.com/r/rust/c") - || chosen.contains("reddit.com/r/rust/d") - || chosen.contains("reddit.com/r/rust/e"); + chosen.contains("https://reddit.com/r/rust/a") || chosen.contains("https://reddit.com/r/rust/b"); + let from_cde = chosen.contains("https://reddit.com/r/rust/c") + || chosen.contains("https://reddit.com/r/rust/d") + || chosen.contains("https://reddit.com/r/rust/e"); assert!( from_ab && from_cde, "expected ranked+unranked attach, got {:?}", @@ -464,40 +464,40 @@ mod tests { #[test] fn suggest_connects_isolate_to_existing_component() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); let mut tree = seed_children( &parent, &[ - "reddit.com/r/rust/a", - "reddit.com/r/rust/b", - "reddit.com/r/rust/c", + "https://reddit.com/r/rust/a", + "https://reddit.com/r/rust/b", + "https://reddit.com/r/rust/c", ], ); let ab = - VoteData::from_recorded(1, "reddit.com/r/rust/a", "reddit.com/r/rust/b", 2, 1).unwrap(); + VoteData::from_recorded(1, "https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b", 2, 1).unwrap(); tree.apply_vote(&parent, ab); let group = tree.get(&parent).unwrap().local_ranking.clone(); let pool = children_of(&tree, &parent); let pair = suggest_next_pair_in_pool(&group, &pool, None).unwrap(); let chosen = pair_set(&pair); - assert!(chosen.contains("reddit.com/r/rust/c")); - assert!(chosen.contains("reddit.com/r/rust/a") || chosen.contains("reddit.com/r/rust/b")); + assert!(chosen.contains("https://reddit.com/r/rust/c")); + assert!(chosen.contains("https://reddit.com/r/rust/a") || chosen.contains("https://reddit.com/r/rust/b")); } #[test] fn suggest_zips_adjacent_ranks_when_tree_complete() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); let mut tree = seed_children( &parent, &[ - "reddit.com/r/rust/a", - "reddit.com/r/rust/b", - "reddit.com/r/rust/c", + "https://reddit.com/r/rust/a", + "https://reddit.com/r/rust/b", + "https://reddit.com/r/rust/c", ], ); for (a, b, l, r) in [ - ("reddit.com/r/rust/a", "reddit.com/r/rust/b", 3, 1), - ("reddit.com/r/rust/a", "reddit.com/r/rust/c", 2, 1), + ("https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b", 3, 1), + ("https://reddit.com/r/rust/a", "https://reddit.com/r/rust/c", 2, 1), ] { let v = VoteData::from_recorded(1, a, b, l, r).unwrap(); tree.apply_vote(&parent, v); @@ -506,26 +506,26 @@ mod tests { let pool = children_of(&tree, &parent); let pair = suggest_next_pair_in_pool(&group, &pool, None).unwrap(); let chosen = pair_set(&pair); - assert!(chosen.contains("reddit.com/r/rust/b")); - assert!(chosen.contains("reddit.com/r/rust/c")); + assert!(chosen.contains("https://reddit.com/r/rust/b")); + assert!(chosen.contains("https://reddit.com/r/rust/c")); } #[test] fn suggest_zip_prefers_1v2_before_2v3_when_both_unvoted() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); let mut tree = seed_children( &parent, &[ - "reddit.com/r/rust/a", - "reddit.com/r/rust/b", - "reddit.com/r/rust/c", - "reddit.com/r/rust/d", + "https://reddit.com/r/rust/a", + "https://reddit.com/r/rust/b", + "https://reddit.com/r/rust/c", + "https://reddit.com/r/rust/d", ], ); for (a, b, l, r) in [ - ("reddit.com/r/rust/c", "reddit.com/r/rust/d", 3, 1), - ("reddit.com/r/rust/b", "reddit.com/r/rust/c", 2, 1), - ("reddit.com/r/rust/a", "reddit.com/r/rust/c", 2, 1), + ("https://reddit.com/r/rust/c", "https://reddit.com/r/rust/d", 3, 1), + ("https://reddit.com/r/rust/b", "https://reddit.com/r/rust/c", 2, 1), + ("https://reddit.com/r/rust/a", "https://reddit.com/r/rust/c", 2, 1), ] { let v = VoteData::from_recorded(1, a, b, l, r).unwrap(); tree.apply_vote(&parent, v); @@ -534,16 +534,16 @@ mod tests { let pool = children_of(&tree, &parent); let pair = suggest_next_pair_in_pool(&group, &pool, None).unwrap(); let chosen = pair_set(&pair); - assert!(chosen.contains("reddit.com/r/rust/a")); - assert!(chosen.contains("reddit.com/r/rust/b")); + assert!(chosen.contains("https://reddit.com/r/rust/a")); + assert!(chosen.contains("https://reddit.com/r/rust/b")); } #[test] fn resolve_pair_picks_from_pool() { - let parent = ItemId::parse("reddit.com/r/rust").unwrap(); - let tree = seed_children(&parent, &["reddit.com/r/rust/a", "reddit.com/r/rust/b"]); + let parent = ItemId::parse("https://reddit.com/r/rust").unwrap(); + let tree = seed_children(&parent, &["https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b"]); let pair = resolve_pair(&tree, &parent, None, None).unwrap(); - let pool: HashSet<_> = ["reddit.com/r/rust/a", "reddit.com/r/rust/b"] + let pool: HashSet<_> = ["https://reddit.com/r/rust/a", "https://reddit.com/r/rust/b"] .into_iter() .collect(); assert!(pool.contains(pair.0.as_str())); diff --git a/server/src/parser.rs b/server/src/parser.rs index 9df2dcc9313f7fe250ce3b6aa167f6ec5d57951f..b2a963dd6cab415576c8d3a9a241966758565bb8 100644 --- a/server/src/parser.rs +++ b/server/src/parser.rs @@ -23,7 +23,7 @@ mod tests { fn parses_short_path() { assert_eq!( parse_reddit_url("r/rust").unwrap().as_str(), - "reddit.com/r/rust" + "https://reddit.com/r/rust" ); } @@ -33,7 +33,7 @@ mod tests { parse_reddit_url("https://www.reddit.com/r/programming/hot") .unwrap() .as_str(), - "reddit.com/r/programming" + "https://reddit.com/r/programming" ); } @@ -43,7 +43,10 @@ mod tests { "https://old.reddit.com/r/AmItheAsshole/comments/1trnvdl/aita_for_cancelling/", ) .unwrap(); - assert_eq!(id.as_str(), "reddit.com/r/amitheasshole/comments/1trnvdl"); + assert_eq!( + id.as_str(), + "https://reddit.com/r/amitheasshole/comments/1trnvdl" + ); } #[test] diff --git a/server/src/path_types.rs b/server/src/path_types.rs index fafd924452f6fd85e7a5b27ed2653a19581e6e56..71ffc01f35c5589686ff05dae3b5610fa01f30ce 100644 --- a/server/src/path_types.rs +++ b/server/src/path_types.rs @@ -1,13 +1,14 @@ use serde::{Deserialize, Serialize}; use std::fmt; -/// Canonical hierarchical identity for any URL/path in the fractal tree. +use crate::url_rules::{looks_like_url, navigable_breadcrumbs, parent_url, resolve_canonical}; + +/// Canonical identity: a real URL (with scheme) or an opaque non-URL key. #[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default)] pub struct ItemId(String); impl ItemId { - /// Parse an already-canonical path (no URL normalization). Empty string is invalid here; - /// use [`Self::root`] for the tree root. + /// Parse an already-canonical id (no normalization). Empty string is invalid; use [`Self::root`]. pub fn parse(s: &str) -> Option { let t = s.trim(); if t.is_empty() { @@ -16,12 +17,12 @@ impl ItemId { Some(Self(t.to_string())) } - /// Build an opaque item key (legacy demo votes, non-URL items). + /// Build an opaque item key (demo votes, non-URL items). pub fn opaque(s: impl Into) -> Self { Self(s.into()) } - /// Root of the internet tree (empty path). + /// Root of the internet tree. pub fn root() -> Self { Self(String::new()) } @@ -34,23 +35,18 @@ impl ItemId { &self.0 } - /// Creates a canonical ID from a raw URL or path. Normalizes domains and - /// trims tracking query params. + /// Canonical URL from a raw pasted or fetched URL. pub fn from_url(raw_url: &str) -> Option { - Self::canonicalize(raw_url).map(Self) + resolve_canonical(raw_url).map(Self) } - /// Normalize strings from forms, events, and Reddit imports into the same - /// stored id shape (e.g. drop post title slug after comment id). + /// Normalize strings from forms, events, and imports into canonical identity. pub fn from_storage(s: &str) -> Option { let t = s.trim(); if t.is_empty() { return None; } - if t.contains("://") || t.starts_with("r/") { - return Self::from_url(t).or_else(|| Self::parse(t)); - } - if t.starts_with("reddit.com/") && t.contains("/comments/") { + if looks_like_url(t) { return Self::from_url(t).or_else(|| Self::parse(t)); } Self::parse(t).or_else(|| Self::from_url(t)) @@ -62,35 +58,52 @@ impl ItemId { if s.is_empty() { return Self::root(); } - Self(format!("reddit.com/r/{s}")) + if looks_like_url(s) || s.contains('/') { + Self::from_storage(s).unwrap_or_else(|| Self::opaque(s)) + } else { + Self(format!("https://reddit.com/r/{s}")) + } } - /// Extract the parent, e.g. `reddit.com/r/aww/comments/1trnvdl` → - /// `reddit.com/r/aww`. + /// Immediate parent scope in the tree. pub fn parent(&self) -> Option { - if self.0.is_empty() { + if self.is_root() { return None; } - + if looks_like_url(self.0.as_str()) { + return parent_url(self.0.as_str()).map(Self); + } let parts: Vec<&str> = self.0.trim_end_matches('/').split('/').collect(); if parts.len() <= 1 { return None; } - - if self.0.contains("/comments/") { - return Some(Self(parts[..parts.len().saturating_sub(2)].join("/"))); - } - Some(Self(parts[..parts.len() - 1].join("/"))) } pub fn segments(&self) -> Vec<&str> { + if self.is_root() { + return vec![]; + } + if let Some(rest) = self.0.strip_prefix("https://") { + return rest.split('/').filter(|s| !s.is_empty()).collect(); + } + if let Some(rest) = self.0.strip_prefix("http://") { + return rest.split('/').filter(|s| !s.is_empty()).collect(); + } self.0.split('/').filter(|s| !s.is_empty()).collect() } - /// Cumulative paths for breadcrumb rendering, e.g. - /// `reddit.com/r/movies` → `["reddit.com", "reddit.com/r", "reddit.com/r/movies"]`. + /// Cumulative navigable paths for breadcrumbs and tree wiring (includes self). pub fn breadcrumb_paths(&self) -> Vec { + if self.is_root() { + return vec![]; + } + if looks_like_url(self.0.as_str()) { + return navigable_breadcrumbs(self.0.as_str()) + .into_iter() + .map(ItemId) + .collect(); + } let segs = self.segments(); let mut paths = Vec::with_capacity(segs.len()); let mut current = String::new(); @@ -111,13 +124,13 @@ impl ItemId { if self.is_root() { return String::new(); } - if self.as_str().contains("://") { - return self.as_str().to_string(); + if self.0.contains("://") { + return self.0.clone(); } if self.segments().first().is_some_and(|s| s.contains('.')) { - format!("https://{}", self.as_str()) + format!("https://{}", self.0) } else { - self.as_str().to_string() + self.0.clone() } } @@ -144,70 +157,6 @@ impl ItemId { pub fn from_browse_uri(path: &str) -> Option { path.strip_prefix("/~/").map(ItemId::from_browse_tail) } - - fn canonicalize(raw: &str) -> Option { - let s = raw.trim(); - if s.is_empty() { - return None; - } - - let owned = if let Some(rest) = s.strip_prefix("r/") { - format!("reddit.com/r/{rest}") - } else if let Some(rest) = s.strip_prefix("/r/") { - format!("reddit.com/r/{rest}") - } else { - s.to_string() - }; - - let (host_path, _query) = split_query(&owned); - let host_path = host_path.trim_end_matches('/'); - - let path = if host_path.contains("://") { - parse_url_host_path(host_path)? - } else if host_path.starts_with("reddit.com") || host_path.starts_with("www.reddit.com") { - normalize_reddit_host_path(host_path) - } else if host_path.contains('/') { - host_path.to_string() - } else { - return None; - }; - - Some(normalize_reddit_path(&path)) - } -} - -fn split_query(s: &str) -> (&str, Option<&str>) { - if let Some((path, q)) = s.split_once('?') { - (path, Some(q)) - } else { - (s, None) - } -} - -fn parse_url_host_path(url: &str) -> Option { - let rest = url - .strip_prefix("https://") - .or_else(|| url.strip_prefix("http://")) - .unwrap_or(url); - let (host, path) = rest.split_once('/').unwrap_or((rest, "")); - let host = normalize_host(host); - if path.is_empty() { - Some(host) - } else { - Some(format!("{host}/{path}")) - } -} - -fn normalize_host(host: &str) -> String { - let h = host - .strip_prefix("www.") - .unwrap_or(host) - .to_ascii_lowercase(); - if h == "old.reddit.com" || h == "new.reddit.com" || h == "reddit.com" { - "reddit.com".to_string() - } else { - h - } } fn normalize_browse_tail(tail: &str) -> String { @@ -215,7 +164,6 @@ fn normalize_browse_tail(tail: &str) -> String { if t.is_empty() { return String::new(); } - // Some HTTP stacks collapse `https://` → `https:/` inside a path segment. if t.starts_with("https:/") && !t.starts_with("https://") { return format!("https://{}", &t[7..]); } @@ -225,33 +173,6 @@ fn normalize_browse_tail(tail: &str) -> String { t.to_string() } -fn normalize_reddit_host_path(s: &str) -> String { - let (host, path) = s.split_once('/').unwrap_or((s, "")); - let host = normalize_host(host); - if path.is_empty() { - host - } else { - format!("{host}/{path}") - } -} - -/// Lowercase subreddit segment, drop listing suffixes, drop title slug after post id. -fn normalize_reddit_path(path: &str) -> String { - let mut parts: Vec = path.split('/').map(str::to_string).collect(); - if parts.len() >= 3 && parts[1] == "r" { - parts[2] = parts[2].to_ascii_lowercase(); - } - if let Some(i) = parts.iter().position(|p| p == "comments") { - if parts.len() > i + 2 { - parts.truncate(i + 2); - } - } else if parts.len() > 3 && parts.get(1).map(|s| s.as_str()) == Some("r") { - // reddit.com/r/{sub}/hot → reddit.com/r/{sub} - parts.truncate(3); - } - parts.join("/") -} - impl fmt::Display for ItemId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.0) @@ -268,43 +189,59 @@ mod tests { "https://old.reddit.com/r/AmItheAsshole/comments/1trnvdl/aita_for_cancelling/", ) .unwrap(); - assert_eq!(id.as_str(), "reddit.com/r/amitheasshole/comments/1trnvdl"); + assert_eq!( + id.as_str(), + "https://reddit.com/r/amitheasshole/comments/1trnvdl" + ); } #[test] fn from_url_strips_query() { let id = ItemId::from_url("https://www.reddit.com/r/rust/?sort=top").unwrap(); - assert_eq!(id.as_str(), "reddit.com/r/rust"); + assert_eq!(id.as_str(), "https://reddit.com/r/rust"); } #[test] fn from_url_short_path() { assert_eq!( ItemId::from_url("r/rust").unwrap().as_str(), - "reddit.com/r/rust" + "https://reddit.com/r/rust" ); } #[test] fn parent_of_post_is_subreddit() { - let id = ItemId::parse("reddit.com/r/aww/comments/1trnvdl").unwrap(); - assert_eq!(id.parent().unwrap().as_str(), "reddit.com/r/aww"); + let id = ItemId::from_url("https://reddit.com/r/aww/comments/1trnvdl").unwrap(); + assert_eq!(id.parent().unwrap().as_str(), "https://reddit.com/r/aww"); } #[test] fn parent_of_subreddit_is_r_segment() { - let id = ItemId::parse("reddit.com/r/movies").unwrap(); - assert_eq!(id.parent().unwrap().as_str(), "reddit.com/r"); + let id = ItemId::from_url("https://reddit.com/r/movies").unwrap(); + assert_eq!(id.parent().unwrap().as_str(), "https://reddit.com/r"); + } + + #[test] + fn breadcrumb_paths_skip_phantom_comments() { + let id = ItemId::from_url("https://reddit.com/r/aww/comments/1trnvdl").unwrap(); + let crumbs = id.breadcrumb_paths(); + let paths: Vec<_> = crumbs.iter().map(|p| p.as_str()).collect(); + assert!(!paths.iter().any(|p| p.ends_with("/comments"))); + assert!(paths.contains(&"https://reddit.com/r/aww")); } #[test] - fn breadcrumb_paths() { - let id = ItemId::parse("reddit.com/r/movies").unwrap(); + fn breadcrumb_paths_subreddit() { + let id = ItemId::from_url("https://reddit.com/r/movies").unwrap(); let crumbs = id.breadcrumb_paths(); let paths: Vec<_> = crumbs.iter().map(|p| p.as_str()).collect(); assert_eq!( paths, - vec!["reddit.com", "reddit.com/r", "reddit.com/r/movies"] + vec![ + "https://reddit.com", + "https://reddit.com/r", + "https://reddit.com/r/movies" + ] ); } @@ -312,33 +249,33 @@ mod tests { fn legacy_scope_maps_to_reddit_sub() { assert_eq!( ItemId::from_legacy_scope("rust").as_str(), - "reddit.com/r/rust" + "https://reddit.com/r/rust" ); assert!(ItemId::from_legacy_scope("").is_root()); } #[test] - fn browse_href_wraps_canonical_path() { - let id = ItemId::parse("reddit.com/r/rust").unwrap(); + fn browse_href_wraps_canonical_url() { + let id = ItemId::from_url("https://reddit.com/r/rust").unwrap(); assert_eq!(id.browse_href(), "/~/https://reddit.com/r/rust"); } #[test] fn from_browse_tail_parses_full_url() { let id = ItemId::from_browse_tail("https://reddit.com/r/AmITheAsshole"); - assert_eq!(id.as_str(), "reddit.com/r/amitheasshole"); + assert_eq!(id.as_str(), "https://reddit.com/r/amitheasshole"); } #[test] fn from_storage_strips_post_title_slug() { let id = ItemId::from_storage("reddit.com/r/rust/comments/aaa/announcing_rust_199").unwrap(); - assert_eq!(id.as_str(), "reddit.com/r/rust/comments/aaa"); + assert_eq!(id.as_str(), "https://reddit.com/r/rust/comments/aaa"); } #[test] fn from_browse_uri_strips_prefix() { let id = ItemId::from_browse_uri("/~/https://reddit.com/r/rust").unwrap(); - assert_eq!(id.as_str(), "reddit.com/r/rust"); + assert_eq!(id.as_str(), "https://reddit.com/r/rust"); } } diff --git a/server/src/projection_apply.rs b/server/src/projection_apply.rs index ebe122e417bda1d9369a53443de93d28213c5a0d..5644557a41b3e9497c7421b444155ae629fa79f1 100644 --- a/server/src/projection_apply.rs +++ b/server/src/projection_apply.rs @@ -19,13 +19,21 @@ use crate::{ storage_schema::{ensure_path_writes, entity_view_writes, vote_writes}, }; -/// Legacy-compatible scope parsing for persisted vote events. +fn parse_event_id(id: &str) -> Result { + ItemId::from_storage(id) + .or_else(|| ItemId::parse(id)) + .ok_or_else(|| EventLogError::Apply(format!("invalid id: {id}"))) +} + +/// Scope key from a vote event (canonicalized at apply time). fn parent_from_event_scope(scope: &str) -> ItemId { - if scope.contains('/') { - ItemId::parse(scope).unwrap_or_else(|| ItemId::from_legacy_scope(scope)) - } else { - ItemId::from_legacy_scope(scope) + let s = scope.trim(); + if s.is_empty() { + return ItemId::root(); } + ItemId::from_storage(s) + .or_else(|| ItemId::parse(s)) + .unwrap_or_else(|| ItemId::from_legacy_scope(s)) } pub fn apply_records( @@ -68,15 +76,11 @@ pub fn apply_records( vote_parents.insert(parent); } Event::NodeEnsured { id } => { - let parsed = ItemId::parse(id) - .or_else(|| ItemId::from_url(id)) - .ok_or_else(|| EventLogError::Apply(format!("invalid node id: {id}")))?; + let parsed = parse_event_id(id)?; ensure_path_writes(&mut batch, &parsed); } Event::EntityImported { id, payload, .. } => { - let parsed = ItemId::parse(id) - .or_else(|| ItemId::from_url(id)) - .ok_or_else(|| EventLogError::Apply(format!("invalid entity id: {id}")))?; + let parsed = parse_event_id(id)?; let view = entity_view_from_payload(&parsed, payload); entity_view_writes(&mut batch, &parsed, view.as_ref()); entity_store @@ -92,7 +96,6 @@ pub fn apply_records( .commit_with(durable::Durability::DisableWal) .map_err(|e| EventLogError::Apply(e.to_string()))?; - // Cap recent-vote windows (idempotent, blind; not part of the cursor batch). for parent in vote_parents { projection_store .trim_recent_votes(&parent) diff --git a/server/src/reddit.rs b/server/src/reddit.rs index 72caf7c33b9dd44ea15b62b59e91227cf96a3431..20b7f9e3f8be39268a1767d09f5cf81eaa6ae0df 100644 --- a/server/src/reddit.rs +++ b/server/src/reddit.rs @@ -183,7 +183,7 @@ pub fn entity_view_from_payload( id: &ItemId, payload: &Value, ) -> Option { - if id.as_str().starts_with("reddit.com") { + if id.as_str().contains("reddit.com") { return parse_reddit_view(id, payload); } None @@ -495,41 +495,59 @@ fn rate_limit_reset_secs(resp: &reqwest::Response) -> u64 { .unwrap_or(5) } +fn reddit_path_segments(id: &ItemId) -> Option> { + let s = id.as_str(); + let rest = s + .strip_prefix("https://reddit.com/") + .or_else(|| s.strip_prefix("http://reddit.com/")) + .or_else(|| s.strip_prefix("reddit.com/"))?; + let segments: Vec = rest + .split('/') + .filter(|p| !p.is_empty()) + .map(str::to_string) + .collect(); + Some(segments) +} + pub fn map_item_to_reddit_api(id: &ItemId, api_base: &str) -> String { - let path = id.as_str(); - if !path.starts_with("reddit.com/") && path != "reddit.com" { - return String::new(); - } + let segments = match reddit_path_segments(id) { + Some(s) => s, + None if matches!( + id.as_str(), + "https://reddit.com" | "http://reddit.com" | "reddit.com" + ) => + { + return String::new(); + } + None => return String::new(), + }; let base = api_base.trim_end_matches('/'); - let segments: Vec<&str> = path.split('/').collect(); - - if let Some(i) = segments.iter().position(|&p| p == "comments") { + if let Some(i) = segments.iter().position(|p| p == "comments") { if segments.len() > i + 1 { - let api_path = segments[1..=i + 1].join("/"); + let api_path = segments[..=i + 1].join("/"); return format!("{base}/{api_path}.json?raw_json=1"); } } - if segments.len() == 3 && segments[1] == "r" { - return format!("{base}/r/{}/about.json?raw_json=1", segments[2]); + if segments.len() == 2 && segments[0] == "r" { + return format!("{base}/r/{}/about.json?raw_json=1", segments[1]); } String::new() } /// Listing URL for a node's children. Currently only subreddits -/// (`reddit.com/r/` → `/r/.json`) expose a child listing. +/// (`https://reddit.com/r/` → `/r/.json`) expose a child listing. pub fn map_children_url(id: &ItemId, api_base: &str) -> String { - let path = id.as_str(); - if !path.starts_with("reddit.com/") { - return String::new(); - } + let segments = match reddit_path_segments(id) { + Some(s) => s, + None => return String::new(), + }; let base = api_base.trim_end_matches('/'); - let segments: Vec<&str> = path.split('/').collect(); - if segments.len() == 3 && segments[1] == "r" { - return format!("{base}/r/{}.json?raw_json=1&limit=25", segments[2]); + if segments.len() == 2 && segments[0] == "r" { + return format!("{base}/r/{}.json?raw_json=1&limit=25", segments[1]); } String::new() } @@ -548,8 +566,8 @@ fn parse_children(_parent: &ItemId, payload: &Value) -> Vec<(ItemId, Value)> { Some(p) if !p.is_empty() => p, _ => continue, }; - let path = format!("reddit.com{}", permalink.trim_end_matches('/')); - if let Some(id) = ItemId::from_storage(&path) { + let raw = format!("https://reddit.com{}", permalink.trim_end_matches('/')); + if let Some(id) = ItemId::from_url(&raw) { out.push((id, child.clone())); } } @@ -683,7 +701,7 @@ mod tests { #[test] fn map_subreddit_about_url() { - let id = ItemId::parse("reddit.com/r/rust").unwrap(); + let id = ItemId::from_url("https://reddit.com/r/rust").unwrap(); assert_eq!( map_item_to_reddit_api(&id, "https://www.reddit.com"), "https://www.reddit.com/r/rust/about.json?raw_json=1" @@ -699,7 +717,8 @@ mod tests { let json = include_str!("../../test/fixtures/reddit/r_rust_about.json"); let v: Value = serde_json::from_str(json).unwrap(); let entity = - entity_view_from_payload(&ItemId::parse("reddit.com/r/rust").unwrap(), &v).unwrap(); + entity_view_from_payload(&ItemId::from_url("https://reddit.com/r/rust").unwrap(), &v) + .unwrap(); assert_eq!(entity.title, "The Rust Programming Language"); } @@ -707,7 +726,8 @@ mod tests { fn parse_post_listing_extracts_thumb_and_full_preview() { let json = include_str!("../../test/fixtures/reddit/post_preview.json"); let v: Value = serde_json::from_str(json).unwrap(); - let id = ItemId::parse("reddit.com/r/nsfw/comments/1tpy6a1/angel_eyes").unwrap(); + let id = + ItemId::from_url("https://reddit.com/r/nsfw/comments/1tpy6a1/angel_eyes").unwrap(); let entity = entity_view_from_payload(&id, &v).unwrap(); assert_eq!(entity.title, "Angel Eyes"); assert!(entity.thumb_url.as_ref().unwrap().contains("width=140")); diff --git a/server/src/reducer.rs b/server/src/reducer.rs index 6578f64a41726845517cdbf59a359c69e0aa56db..5179ddeca7a7cb0fb92dfc4aa9d5a80bd9125611 100644 --- a/server/src/reducer.rs +++ b/server/src/reducer.rs @@ -248,16 +248,18 @@ mod from_recorded_tests { #[test] fn ensure_path_wires_children() { let mut tree = GlobalTree::new(); - let id = ItemId::parse("reddit.com/r/rust").unwrap(); + let id = ItemId::from_url("https://reddit.com/r/rust").unwrap(); tree.ensure_path(&id); let root = tree.get(&ItemId::root()).unwrap(); assert!(root .children - .contains(&ItemId::parse("reddit.com").unwrap())); - let reddit = tree.get(&ItemId::parse("reddit.com").unwrap()).unwrap(); + .contains(&ItemId::from_url("https://reddit.com").unwrap())); + let reddit = tree + .get(&ItemId::from_url("https://reddit.com").unwrap()) + .unwrap(); assert!(reddit .children - .contains(&ItemId::parse("reddit.com/r").unwrap())); + .contains(&ItemId::from_url("https://reddit.com/r").unwrap())); let sub = tree.get(&id).unwrap(); assert_eq!(sub.id, id); } diff --git a/server/src/render/reddit.rs b/server/src/render/reddit.rs index 7f840aa33b734a31d8cf3341a0581c8bcb9bbcf3..595e202436040b0bfc419e68f083f94757ba5d0c 100644 --- a/server/src/render/reddit.rs +++ b/server/src/render/reddit.rs @@ -9,7 +9,7 @@ use crate::{ }; pub fn is_reddit_post(id: &ItemId) -> bool { - id.as_str().starts_with("reddit.com/") && id.as_str().contains("/comments/") + id.as_str().contains("reddit.com/") && id.as_str().contains("/comments/") } /// Post detail card (inside [`crate::fetch::html::entity_panel`]). diff --git a/server/src/state.rs b/server/src/state.rs index 78126f08d90f8069a586279d79258c27a9f9f7a4..513b329ef45fc332e63b3f8ed8498a1f59feb07c 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -217,15 +217,21 @@ impl AppState { ratio_right: i32, ) -> Result<(), String> { let ts = crate::html::now_ms(); - let vote = VoteData::from_recorded(ts, a, b, ratio_left, ratio_right) - .ok_or_else(|| "invalid vote: need two distinct non-empty items".to_string())?; + let a_raw = a.trim(); + let b_raw = b.trim(); + if a_raw.is_empty() || b_raw.is_empty() || a_raw == b_raw { + return Err("invalid vote: need two distinct non-empty items".to_string()); + } + // Validate items canonicalize (or are opaque keys) before append. + let _ = VoteData::from_recorded(ts, a_raw, b_raw, ratio_left, ratio_right) + .ok_or_else(|| "invalid vote: need two distinct parseable items".to_string())?; let event = Event::VoteRecorded { ts, - a: vote.a.as_str().to_string(), - b: vote.b.as_str().to_string(), - ratio_left: vote.ratio_left, - ratio_right: vote.ratio_right, + a: a_raw.to_string(), + b: b_raw.to_string(), + ratio_left, + ratio_right, scope: parent.as_str().to_string(), }; @@ -253,7 +259,7 @@ mod tests { let log = EventLog::new(log_path.to_string_lossy().into_owned()); let payload = json!({"kind":"t5","data":{"title":"Rust","display_name":"rust"}}); let event = Event::EntityImported { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), ts: 1, payload: payload.clone(), }; @@ -266,14 +272,14 @@ mod tests { .await .unwrap(); let tree = projection_store - .scope_tree(&ItemId::parse("reddit.com/r/rust").unwrap()) + .scope_tree(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .unwrap(); let node = tree - .get(&ItemId::parse("reddit.com/r/rust").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .unwrap(); assert_eq!(node.data.as_ref().unwrap().title, "Rust"); let stored = entity_store - .get(&ItemId::parse("reddit.com/r/rust").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .unwrap() .unwrap(); assert_eq!(stored["data"]["display_name"], "rust"); @@ -289,13 +295,13 @@ mod tests { event_record( 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, ), event_record( 2, Event::EntityImported { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), ts: 2, payload: payload.clone(), }, @@ -325,7 +331,7 @@ mod tests { &[event_record( 1, Event::NodeEnsured { - id: "reddit.com/r/stale".into(), + id: "https://reddit.com/r/stale".into(), }, )], ) @@ -352,11 +358,11 @@ mod tests { let root = tree.get(&ItemId::root()).unwrap(); assert!(root.children.contains(&ItemId::parse("alpha").unwrap())); assert!(projection_store - .load_node(&ItemId::parse("reddit.com/r/stale").unwrap()) + .load_node(&ItemId::parse("https://reddit.com/r/stale").unwrap()) .unwrap() .is_none()); let stored = entity_store - .get(&ItemId::parse("reddit.com/r/rust").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .unwrap() .unwrap(); assert_eq!(stored["data"]["display_name"], "rust"); @@ -370,7 +376,7 @@ mod tests { log.append(&event_record( 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )) .await @@ -387,7 +393,7 @@ mod tests { &[event_record( 2, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )], ) @@ -454,7 +460,7 @@ mod tests { port: 0, }) .await; - let id = ItemId::parse("reddit.com/r/rust").unwrap(); + let id = ItemId::parse("https://reddit.com/r/rust").unwrap(); state.ensure_node(&id).await.unwrap(); @@ -465,11 +471,11 @@ mod tests { let projected = state.projection_store.load_tree().unwrap(); assert!(projected.get(&id).is_some()); let reddit = projected - .get(&ItemId::parse("reddit.com").unwrap()) + .get(&ItemId::from_url("https://reddit.com").unwrap()) .unwrap(); assert!(reddit .children - .contains(&ItemId::parse("reddit.com/r").unwrap())); + .contains(&ItemId::from_url("https://reddit.com/r").unwrap())); } #[tokio::test] @@ -510,7 +516,7 @@ mod tests { log.append(&event_record( 1, Event::NodeEnsured { - id: "reddit.com/r/rust".into(), + id: "https://reddit.com/r/rust".into(), }, )) .await @@ -518,7 +524,7 @@ mod tests { log.append(&event_record( 2, Event::NodeEnsured { - id: "reddit.com/r/python".into(), + id: "https://reddit.com/r/python".into(), }, )) .await @@ -544,13 +550,13 @@ mod tests { 2 ); let tree = second - .scope_tree(&ItemId::parse("reddit.com/r/rust").unwrap()) + .scope_tree(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .unwrap(); assert!(tree - .get(&ItemId::parse("reddit.com/r/rust").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) .is_some()); assert!(tree - .get(&ItemId::parse("reddit.com/r/python").unwrap()) + .get(&ItemId::parse("https://reddit.com/r/python").unwrap()) .is_none()); } @@ -611,7 +617,7 @@ mod tests { #[test] fn parse_item_param_from_url() { let id = parse_item_param("https://reddit.com/r/rust"); - assert_eq!(id.as_str(), "reddit.com/r/rust"); + assert_eq!(id.as_str(), "https://reddit.com/r/rust"); } #[test] diff --git a/server/src/url_rules/engine.rs b/server/src/url_rules/engine.rs new file mode 100644 index 0000000000000000000000000000000000000000..e29b6b48c08deb7bffe031b1e542b1e25a7bef15 --- /dev/null +++ b/server/src/url_rules/engine.rs @@ -0,0 +1,187 @@ +//! Composable URL normalization primitives. + +use std::collections::HashMap; + +use url::Url; + +/// Mutable URL view used by rule combinators before serializing to a canonical string. +#[derive(Debug, Clone)] +pub struct ParsedUrl { + pub scheme: String, + pub host: String, + pub path_segments: Vec, + pub query: HashMap, + pub fragment: Option, +} + +impl ParsedUrl { + pub fn parse(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + let with_scheme = if trimmed.contains("://") { + trimmed.to_string() + } else if trimmed.starts_with("r/") || trimmed.starts_with("/r/") { + let rest = trimmed.trim_start_matches('/').trim_start_matches("r/"); + format!("https://reddit.com/r/{rest}") + } else if trimmed.contains('.') && !trimmed.starts_with('/') { + format!("https://{trimmed}") + } else { + trimmed.to_string() + }; + + let url = Url::parse(&with_scheme).ok()?; + let host = url.host_str()?.to_string(); + let path_segments: Vec = url + .path_segments() + .map(|segs| segs.filter(|s| !s.is_empty()).map(str::to_string).collect()) + .unwrap_or_default(); + + let mut query = HashMap::new(); + for (k, v) in url.query_pairs() { + query.insert(k.into_owned(), v.into_owned()); + } + + Some(Self { + scheme: url.scheme().to_string(), + path_segments, + query, + fragment: url.fragment().map(str::to_string), + host, + }) + } + + pub fn with_path_segments(&self, segments: &[String]) -> Self { + let mut u = self.clone(); + u.path_segments = segments.to_vec(); + u + } + + pub fn to_url(&self) -> Option { + let mut url = if self.path_segments.is_empty() { + Url::parse(&format!("{}://{}", self.scheme, self.host)).ok()? + } else { + let path = format!("/{}", self.path_segments.join("/")); + Url::parse(&format!("{}://{}{}", self.scheme, self.host, path)).ok()? + }; + if !self.query.is_empty() { + let mut pairs: Vec<_> = self.query.iter().collect(); + pairs.sort_by(|a, b| a.0.cmp(b.0)); + url.query_pairs_mut().clear(); + for (k, v) in pairs { + url.query_pairs_mut().append_pair(k, v); + } + } + if let Some(ref frag) = self.fragment { + url.set_fragment(Some(frag)); + } + Some(url) + } + + pub fn canonical_string(&self) -> Option { + let url = self.to_url()?; + let mut s = url.to_string(); + if self.path_segments.is_empty() { + s = s.trim_end_matches('/').to_string(); + } + Some(s) + } +} + +pub fn force_https(u: &mut ParsedUrl) { + if u.scheme == "http" { + u.scheme = "https".to_string(); + } +} + +pub fn drop_fragment(u: &mut ParsedUrl) { + u.fragment = None; +} + +pub fn strip_www(u: &mut ParsedUrl) { + if u.host.starts_with("www.") { + u.host = u.host[4..].to_string(); + } +} + +pub fn lowercase_host(u: &mut ParsedUrl) { + u.host = u.host.to_ascii_lowercase(); +} + +pub fn lowercase_path(u: &mut ParsedUrl) { + for seg in &mut u.path_segments { + *seg = seg.to_ascii_lowercase(); + } +} + +pub fn clear_query(u: &mut ParsedUrl) { + u.query.clear(); +} + +pub fn keep_only_query(u: &mut ParsedUrl, keys: &[&str]) { + u.query + .retain(|k, _| keys.iter().any(|want| want == &k.as_str())); +} + +pub fn strip_tracking_params(u: &mut ParsedUrl) { + u.query.retain(|k, _| { + let lower = k.to_ascii_lowercase(); + !(lower.starts_with("utm_") + || matches!( + lower.as_str(), + "fbclid" | "gclid" | "ref" | "ref_src" | "ref_source" | "mc_cid" | "mc_eid" + )) + }); +} + +pub fn truncate_after_segment(u: &mut ParsedUrl, name: &str, keep: usize) { + if let Some(i) = u.path_segments.iter().position(|s| s == name) { + let end = (i + 1 + keep).min(u.path_segments.len()); + u.path_segments.truncate(end); + } +} + +pub fn drop_listing_suffix(u: &mut ParsedUrl, suffixes: &[&str]) { + if u.path_segments.len() >= 3 && u.path_segments.first().map(String::as_str) == Some("r") { + if let Some(last) = u.path_segments.last() { + if suffixes.iter().any(|s| *s == last.as_str()) { + u.path_segments.pop(); + } + } + } +} + +pub fn normalize_reddit_host(u: &mut ParsedUrl) { + if matches!( + u.host.as_str(), + "old.reddit.com" | "new.reddit.com" | "www.reddit.com" + ) { + u.host = "reddit.com".to_string(); + } +} + +pub fn rewrite_youtu_be(u: &mut ParsedUrl) { + if u.host == "youtu.be" && u.path_segments.len() == 1 { + let id = u.path_segments[0].clone(); + u.host = "youtube.com".to_string(); + u.path_segments = vec!["watch".to_string()]; + u.query.insert("v".to_string(), id); + } +} + +pub fn rewrite_youtube_shorts(u: &mut ParsedUrl) { + if u.host == "youtube.com" && u.path_segments.first().map(String::as_str) == Some("shorts") { + if let Some(id) = u.path_segments.get(1).cloned() { + u.path_segments = vec!["watch".to_string()]; + u.query.insert("v".to_string(), id); + } + } +} + +pub fn normalize_youtube_host(u: &mut ParsedUrl) { + if matches!(u.host.as_str(), "m.youtube.com" | "www.youtube.com") { + u.host = "youtube.com".to_string(); + } +} diff --git a/server/src/url_rules/mod.rs b/server/src/url_rules/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..03d53bd3e82d704a01ba3fd8dd02b7d31422c0de --- /dev/null +++ b/server/src/url_rules/mod.rs @@ -0,0 +1,13 @@ +//! URL canonicalization and hierarchy rules for [`crate::path_types::ItemId`]. + +mod engine; +mod registry; + +pub use registry::{ + canonicalize_raw, looks_like_url, navigable_breadcrumbs, parent_url, resolve_id, CanonicalResult, +}; + +/// Resolve raw input to canonical URL. +pub fn resolve_canonical(raw: &str) -> Option { + canonicalize_raw(raw.trim()).map(|r| r.canonical) +} diff --git a/server/src/url_rules/registry.rs b/server/src/url_rules/registry.rs new file mode 100644 index 0000000000000000000000000000000000000000..14514e9af8385fb2b9b2f35eb9ee14d453d4b97c --- /dev/null +++ b/server/src/url_rules/registry.rs @@ -0,0 +1,235 @@ +//! Per-domain canonicalization and hierarchy rules. + +use std::collections::HashSet; + +use super::engine::{ + clear_query, drop_fragment, drop_listing_suffix, force_https, keep_only_query, lowercase_host, + lowercase_path, normalize_reddit_host, normalize_youtube_host, rewrite_youtu_be, + rewrite_youtube_shorts, strip_tracking_params, strip_www, truncate_after_segment, ParsedUrl, +}; + +/// Result of canonicalizing a raw URL string. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CanonicalResult { + pub canonical: String, + /// When the input normalizes to a different string, the original is an alias. + pub alias_of: Option, +} + +fn apply_global(u: &mut ParsedUrl) { + force_https(u); + drop_fragment(u); + strip_www(u); + lowercase_host(u); + strip_tracking_params(u); +} + +fn normalize_reddit(u: &mut ParsedUrl) { + normalize_reddit_host(u); + lowercase_path(u); + truncate_after_segment(u, "comments", 1); + drop_listing_suffix(u, &["hot", "top", "new", "rising", "controversial"]); + clear_query(u); +} + +fn normalize_youtube(u: &mut ParsedUrl) { + rewrite_youtu_be(u); + normalize_youtube_host(u); + rewrite_youtube_shorts(u); + keep_only_query(u, &["v", "list"]); +} + +fn normalize_default(_u: &mut ParsedUrl) { + // Global rules only. +} + +fn domain_key(host: &str) -> &'static str { + if host == "reddit.com" || host.ends_with(".reddit.com") { + "reddit.com" + } else if host == "youtube.com" || host == "youtu.be" { + "youtube.com" + } else { + "default" + } +} + +fn normalize_for_host(u: &mut ParsedUrl) { + apply_global(u); + match domain_key(&u.host) { + "reddit.com" => normalize_reddit(u), + "youtube.com" => normalize_youtube(u), + _ => normalize_default(u), + } +} + +/// Structural path segments that must not become standalone tree nodes when more path follows. +fn structural_trailing(host: &str) -> &'static [&'static str] { + match domain_key(host) { + "reddit.com" => &["comments"], + _ => &[], + } +} + +/// Canonicalize a raw URL. Returns `None` if the input is not URL-like. +pub fn canonicalize_raw(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + let mut u = ParsedUrl::parse(trimmed)?; + let input_snapshot = u.canonical_string()?; + normalize_for_host(&mut u); + let canonical = u.canonical_string()?; + let alias_of = if input_snapshot != canonical { + Some(trimmed.to_string()) + } else { + None + }; + Some(CanonicalResult { + canonical, + alias_of, + }) +} + +/// Resolve a stored or event id string to its canonical URL identity. +pub fn resolve_id(raw: &str) -> Option { + canonicalize_raw(raw).map(|r| r.canonical) +} + +/// Navigable ancestor URLs from domain root up to and including `canonical` (full URLs). +pub fn navigable_breadcrumbs(canonical: &str) -> Vec { + let Some(u) = ParsedUrl::parse(canonical) else { + return vec![canonical.to_string()]; + }; + let structural: HashSet<&str> = structural_trailing(&u.host).iter().copied().collect(); + let n = u.path_segments.len(); + let mut out = Vec::new(); + + // Domain root (no path segments). + if let Some(base) = u.with_path_segments(&[]).canonical_string() { + out.push(base); + } + + for i in 0..n { + let segs: Vec = u.path_segments[..=i].to_vec(); + let is_last = i == n - 1; + let seg = u.path_segments[i].as_str(); + if structural.contains(seg) && !is_last { + continue; + } + if let Some(url) = u.with_path_segments(&segs).canonical_string() { + if out.last() != Some(&url) { + out.push(url); + } + } + } + out +} + +/// Immediate parent scope URL, or `None` for tree root / opaque single-segment ids. +pub fn parent_url(canonical: &str) -> Option { + let crumbs = navigable_breadcrumbs(canonical); + if crumbs.len() <= 1 { + None + } else { + crumbs.get(crumbs.len() - 2).cloned() + } +} + +/// True when `raw` looks like a URL (has scheme or host-like shape). +pub fn looks_like_url(raw: &str) -> bool { + let t = raw.trim(); + t.contains("://") + || t.starts_with("r/") + || t.starts_with("/r/") + || (t.contains('.') && t.contains('/')) + || t.starts_with("reddit.com") + || t.starts_with("www.") + || t.starts_with("youtu.be/") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reddit_post_drops_slug_and_normalizes_host() { + let r = canonicalize_raw( + "https://old.reddit.com/r/AmItheAsshole/comments/1trnvdl/aita_for_cancelling/", + ) + .unwrap(); + assert_eq!( + r.canonical, + "https://reddit.com/r/amitheasshole/comments/1trnvdl" + ); + } + + #[test] + fn reddit_strips_query_and_listing() { + assert_eq!( + canonicalize_raw("https://www.reddit.com/r/rust/?sort=top") + .unwrap() + .canonical, + "https://reddit.com/r/rust" + ); + assert_eq!( + canonicalize_raw("https://www.reddit.com/r/programming/hot") + .unwrap() + .canonical, + "https://reddit.com/r/programming" + ); + } + + #[test] + fn reddit_short_path() { + assert_eq!( + canonicalize_raw("r/rust").unwrap().canonical, + "https://reddit.com/r/rust" + ); + } + + #[test] + fn reddit_breadcrumbs_skip_phantom_comments() { + let post = "https://reddit.com/r/aww/comments/1trnvdl"; + let crumbs = navigable_breadcrumbs(post); + assert!(!crumbs.iter().any(|c| c.ends_with("/comments"))); + assert_eq!( + crumbs.last().map(String::as_str), + Some(post) + ); + assert!(crumbs.contains(&"https://reddit.com/r/aww".to_string())); + } + + #[test] + fn reddit_parent_of_post_is_subreddit() { + assert_eq!( + parent_url("https://reddit.com/r/aww/comments/1trnvdl").as_deref(), + Some("https://reddit.com/r/aww") + ); + } + + #[test] + fn youtube_youtu_be_and_watch_same_canonical() { + let a = canonicalize_raw("https://youtu.be/dQw4w9WgXcQ").unwrap().canonical; + let b = canonicalize_raw("https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=10").unwrap(); + assert_eq!(a, b.canonical); + assert_eq!(a, "https://youtube.com/watch?v=dQw4w9WgXcQ"); + } + + #[test] + fn legacy_schemeless_upgrades() { + assert_eq!( + canonicalize_raw("reddit.com/r/rust/comments/aaa/announcing_rust_199") + .unwrap() + .canonical, + "https://reddit.com/r/rust/comments/aaa" + ); + } + + #[test] + fn alias_recorded_when_input_differs() { + let r = canonicalize_raw("https://youtu.be/abc123").unwrap(); + assert_eq!(r.canonical, "https://youtube.com/watch?v=abc123"); + assert!(r.alias_of.is_some()); + } +} Side B — contributor: tommy-mor Side B — commit message: [4e327784] deploy live constitution dashboard Expose auditable progress and event streaming, configure the production roots and runtime, and make tested main-branch commits the deployment authority. Co-authored-by: Cursor Side B — unified diff (full patch): diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..c9d63a722beba0a0297fc853089332c460ab78dd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.venv +.hypothesis +__pycache__ +tests +*.json +*.bsp diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000000000000000000000000000000000000..76dcdf82d53177c1e47d86b23a54523239d232a6 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,49 @@ +name: Test and deploy + +on: + push: + branches: [main] + +concurrency: + group: production + cancel-in-progress: false + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Run Python tests + run: uv run pytest -q + + - name: Install Babashka + run: | + curl -fsSL https://raw.githubusercontent.com/babashka/babashka/master/install \ + | sudo bash -s -- --dir /usr/local/bin + + - name: Run process integration tests + run: bb TEST.sh + + deploy: + needs: test + runs-on: ubuntu-latest + environment: + name: production + url: https://token.slug.social + steps: + - uses: actions/checkout@v4 + + - uses: superfly/flyctl-actions/setup-flyctl@master + + - name: Deploy to Fly + run: flyctl deploy --remote-only + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c9a5c00782371c19ad5ab5c58cf6f5a8ffec0141 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-install-project + +COPY constitution.py ./ + +ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONUNBUFFERED="1" + +EXPOSE 8080 +CMD ["python", "constitution.py"] diff --git a/constitution.py b/constitution.py index 4bee9f83663ab7fb36db95129b92b64b4ef57258..a58257e1881b21d1d6fa8e68a3faa222e4f661ef 100644 --- a/constitution.py +++ b/constitution.py @@ -24,12 +24,12 @@ A daily GitHub Action backs up the JSONL ledger to the same repo. Run: uv run constitution.py """ -from decimal import Decimal, getcontext +from decimal import Decimal, getcontext, DefaultContext from datetime import datetime, timezone from fastapi import FastAPI, Request, Response from fastapi.responses import PlainTextResponse, HTMLResponse from starlette.middleware.sessions import SessionMiddleware -import json, time, os, asyncio, httpx, pathlib, subprocess, hashlib, re, fcntl +import json, time, os, asyncio, httpx, pathlib, subprocess, hashlib, re, fcntl, base64 import sympy as sp # type: ignore[reportMissingImports] from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential from evaleval import ( @@ -37,6 +37,7 @@ from evaleval import ( exec_event, One, Two, Three, Selector, MORPH, PREPEND, ) +DefaultContext.prec = 50 getcontext().prec = 50 app = FastAPI() @@ -129,14 +130,47 @@ OPENROUTER_BASE_URL = os.environ.get("OPENROUTER_BASE_URL", "https://openrouter. # using the exact same source; their normalized values are committed to every # discovery event. DEFAULT_REPOSITORIES = [ + { + "id": "constitution", + "url": "https://github.com/sortersocial/constitution.git", + "refs": ["refs/heads/**"], + }, { "id": "slug", - "url": "https://github.com/tommy-mor/slug.git", + "url": "https://github.com/sortersocial/slug.git", + "refs": ["refs/heads/**"], + }, + { + "id": "sorter", + "url": "https://github.com/sorterisntonline/sorter.git", + "refs": ["refs/heads/**"], + }, + { + "id": "sorter2", + "url": "https://github.com/sortersocial/sorter2.git", + "refs": ["refs/heads/**"], + }, + { + "id": "sorter-oldest", + "url": "https://github.com/tommy-mor/sorter.git", "refs": ["refs/heads/**"], }, ] DEFAULT_CONTRIBUTORS = { "tommy-mor": ["thmorriss@gmail.com"], + "christopher-whitman": [ + "chris@cwwhitman.com", + "7566903+cwwhitman@users.noreply.github.com", + ], + "jake-chvatal": [ + "jake+github@uln.industries", + "jakechvatal@gmail.com", + "jake@isnt.online", + ], + "lara": ["me@lara.lv"], + "nat-reid": ["nathanielreid@gmail.com"], + "zod": ["jason.p.mcel@gmail.com", "me@zod.tf"], + "jovan": ["jovan@slug.social", "jovan@getcivicai.com"], } REPOSITORIES = json.loads( @@ -147,6 +181,7 @@ CONTRIBUTORS = json.loads( ) GIT_MIRROR_DIR = pathlib.Path(os.environ.get("GIT_MIRROR_DIR", "/data/git")) GIT_TIMEOUT_SECONDS = int(os.environ.get("GIT_TIMEOUT_SECONDS", "120")) +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") # Council model IDs: slug.social garden rank under this parent (bodies = OpenRouter URLs), then top-up from OpenRouter list. SLUG_SOCIAL_BASE_URL = os.environ.get("SLUG_SOCIAL_BASE_URL", "https://slug.social").rstrip("/") @@ -661,20 +696,30 @@ def _git(repo: pathlib.Path | None, *args: str, input_bytes: bytes | None = None if repo is not None: command += ["-C", str(repo)] command += list(args) + git_env = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_NO_REPLACE_OBJECTS": "1", + "LC_ALL": "C", + "TZ": "UTC", + } + if GITHUB_TOKEN: + credential = base64.b64encode( + f"x-access-token:{GITHUB_TOKEN}".encode() + ).decode() + git_env.update({ + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "http.https://github.com/.extraHeader", + "GIT_CONFIG_VALUE_0": f"Authorization: Basic {credential}", + }) try: result = subprocess.run( command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env={ - **os.environ, - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_NO_REPLACE_OBJECTS": "1", - "LC_ALL": "C", - "TZ": "UTC", - }, + env=git_env, timeout=GIT_TIMEOUT_SECONDS, check=False, ) @@ -844,9 +889,15 @@ def _build_discovery(epoch_n: int, boundary_ms: int, events: list) -> GitDiscove canonical_location = min( locations[qualified_oid], key=lambda x: (x[0], x[1]) ) + # One commit may be reachable from dozens of refs in the same mirror. + # Verify its object once per repository, not once per source ref. + object_locations = { + (str(m), raw_oid): (m, raw_oid) + for _, _, m, raw_oid in locations[qualified_oid] + } object_hashes = { hashlib.sha256(_git(m, "cat-file", "commit", raw_oid)).hexdigest() - for _, _, m, raw_oid in locations[qualified_oid] + for m, raw_oid in object_locations.values() } if len(object_hashes) != 1: raise RuntimeError(f"conflicting Git objects share OID {qualified_oid}") @@ -994,11 +1045,55 @@ async def discover_repositories(epoch_n: int, boundary_ms: int) -> GitDiscovery: SSE_CLIENTS = [] +AUDIT_HISTORY = [] +AUDIT_SEQUENCE = 0 +PROCESS_STATE = { + "running": False, + "phase": "idle", + "progress": 100, + "message": "Waiting for the next epoch", +} + + +def _sse_event(event_name: str, payload: dict) -> str: + return ( + f"event: {event_name}\n" + f"data: {json.dumps(payload, separators=(',', ':'))}\n\n" + ) + + +async def broadcast_audit( + kind: str, + message: str, + *, + progress: int | None = None, + phase: str | None = None, +) -> dict: + global AUDIT_SEQUENCE + AUDIT_SEQUENCE += 1 + if progress is not None: + PROCESS_STATE["progress"] = max(0, min(100, int(progress))) + if phase is not None: + PROCESS_STATE["phase"] = phase + PROCESS_STATE["message"] = message + payload = { + "id": AUDIT_SEQUENCE, + "timestamp_ms": int(time.time() * 1000), + "kind": kind, + "message": message, + **PROCESS_STATE, + } + AUDIT_HISTORY.append(payload) + del AUDIT_HISTORY[:-200] + wire = _sse_event("audit", payload) + for queue in list(SSE_CLIENTS): + await queue.put(wire) + return payload async def broadcast_js(js: str): """Send a JS snippet to all connected SSE clients.""" - for queue in SSE_CLIENTS: + for queue in list(SSE_CLIENTS): await queue.put(js) @@ -1006,10 +1101,29 @@ async def rank_commits(commits: list[dict]): if not commits: return {}, [] - models = await fetch_top_models(n=3) contributors = sorted(set(c["contributor"] for c in commits)) - if len(contributors) > 1 and not models: + if len(contributors) == 1: + await broadcast_audit( + "ranking", + f"Only {contributors[0]} is eligible; rank is 1.0", + progress=90, + phase="finalizing", + ) + return {contributors[0]: Decimal("1")}, [] + if not (OPENROUTER_API_KEY or "").strip(): + raise RuntimeError( + "OPENROUTER_API_KEY is required when multiple contributors need ranking" + ) + + models = await fetch_top_models(n=3) + if not models: raise RuntimeError("no council models available for contributor ranking") + await broadcast_audit( + "council", + f"Council selected: {', '.join(models)}", + progress=35, + phase="ranking", + ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ ["div.log-council", f"Council: {', '.join(models)} — {len(commits)} commits"] ])) @@ -1035,6 +1149,11 @@ async def rank_commits(commits: list[dict]): async def compare_fn(i, j): a1, a2 = authors[i], authors[j] + await broadcast_audit( + "comparison", + f"Comparing {a1} with {a2}", + phase="ranking", + ) await broadcast_js(exec_event(Three[Selector("#emission-status")][MORPH][ ["div#emission-status", f"Comparing {a1} vs {a2}…"] ])) @@ -1050,6 +1169,11 @@ async def rank_commits(commits: list[dict]): if winner_weight <= 0 or loser_weight <= 0: raise ValueError("ratio weights must be positive") results.append((w, l, winner_weight, loser_weight)) + await broadcast_audit( + "vote", + f"{model}: {authors[w]} over {authors[l]} ({result['ratio']})", + phase="ranking", + ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ ["div.log-vote", ["span.model", model], " — ", @@ -1059,6 +1183,11 @@ async def rank_commits(commits: list[dict]): ] ])) except Exception as e: + await broadcast_audit( + "error", + f"{model} failed: {e}", + phase="error", + ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ ["div.log-error", f"⚠ {model}: {e}"] ])) @@ -1068,8 +1197,13 @@ async def rank_commits(commits: list[dict]): async def progress_fn(ev): if ev["phase"] == "spanning_tree": label = f"Spanning tree: {ev['step']}/{ev['total']}" + percent = 35 + round(35 * ev["step"] / max(ev["total"], 1)) else: label = f"Zip pass {ev['pass']}: {ev['step']}/{ev['total']}" + percent = 70 + round(20 * ev["step"] / max(ev["total"], 1)) + await broadcast_audit( + "progress", label, progress=percent, phase="ranking" + ) await broadcast_js(exec_event(Three[Selector("#emission-status")][MORPH][ ["div#emission-status", label] ])) @@ -1083,6 +1217,12 @@ async def rank_commits(commits: list[dict]): scores = rank_centrality(pairs) ranking = {authors[i]: Decimal(str(scores[i])) for i in range(len(authors))} ranking_rows = sorted(ranking.items(), key=lambda x: x[1], reverse=True) + await broadcast_audit( + "ranking", + "Ranking: " + ", ".join(f"{a} {s:.4f}" for a, s in ranking_rows), + progress=90, + phase="finalizing", + ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ ["div.log-ranking", ["b", "Ranking: "], @@ -1104,11 +1244,33 @@ def pool_remaining(events: list) -> Decimal: async def run_emission(epoch_n, boundary_ms): + PROCESS_STATE["running"] = True + await broadcast_audit( + "start", + f"Epoch {epoch_n} emission started", + progress=2, + phase="starting", + ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ ["div.log-start", f"⚡ Epoch {epoch_n} emission started"] ])) + await broadcast_audit( + "discovery", + "Fetching configured repositories and snapshotting refs", + progress=8, + phase="discovery", + ) discovery = await discover_repositories(epoch_n, boundary_ms) + await broadcast_audit( + "discovery", + ( + f"Discovered {len(discovery.observations)} new commits; " + f"{len(discovery.commits)} are eligible" + ), + progress=30, + phase="discovery", + ) ranking, models = await rank_commits(discovery.commits) def make_emission(events): @@ -1146,6 +1308,13 @@ async def run_emission(epoch_n, boundary_ms): entry = await store.atomic(make_emission) if entry: + PROCESS_STATE["running"] = False + await broadcast_audit( + "complete", + f"Epoch {entry.epoch} complete; emitted {entry.total_emitted} SLG", + progress=100, + phase="idle", + ) await broadcast_js(exec_event(Three[Selector("#emission-log")][PREPEND][ ["div.log-amount", f"Pool {entry.pool_before} → emit {entry.total_emitted} → {entry.pool_after}"] @@ -1189,13 +1358,24 @@ async def distribute_usdc(holdings, treasury_balance): async def epoch_loop(): while True: epoch_n, current_start, next_boundary = current_epoch() + processed = {e.epoch for e in store.read() if isinstance(e, Emission)} + if epoch_n >= 0 and epoch_n not in processed: + try: + await run_emission(epoch_n, current_start) + except Exception as exc: + PROCESS_STATE["running"] = False + await broadcast_audit( + "error", + f"Epoch {epoch_n} failed: {exc}; retrying in 60 seconds", + phase="error", + ) + print(f"epoch {epoch_n} emission failed: {exc}", flush=True) + await asyncio.sleep(60) + continue + now = int(time.time() * 1000) wait_ms = next_boundary - now - if wait_ms <= 0: - processed = {e.epoch for e in store.read() if isinstance(e, Emission)} - if epoch_n not in processed and epoch_n >= 0: - await run_emission(epoch_n, current_start) await asyncio.sleep(60) elif wait_ms < 86_400_000: await broadcast_js(exec_event(Three[Selector("#emission-status")][MORPH][ @@ -1243,6 +1423,36 @@ async def get_ranking(): return {"ranking": latest.ranking, "epoch": latest.epoch} +@app.get("/api/status") +async def get_status(): + events = store.read() + discoveries = [e for e in events if isinstance(e, GitDiscovery)] + emissions = [e for e in events if isinstance(e, Emission)] + return { + **PROCESS_STATE, + "epoch": current_epoch()[0], + "openrouter_configured": bool((OPENROUTER_API_KEY or "").strip()), + "sse_clients": len(SSE_CLIENTS), + "latest_discovery": ( + { + "epoch": discoveries[-1].epoch, + "snapshot_id": discoveries[-1].snapshot_id, + "observations": len(discoveries[-1].observations), + "eligible_commits": len(discoveries[-1].commits), + } + if discoveries else None + ), + "latest_emission": ( + { + "epoch": emissions[-1].epoch, + "total_emitted": emissions[-1].total_emitted, + "ranking": emissions[-1].ranking, + } + if emissions else None + ), + } + + @app.get("/api/contributor/{github_username}") async def get_contributor(github_username: str): history = [ @@ -1306,13 +1516,6 @@ async def test_emit(): # =========================================================================== # §9. SSE — live audit stream of the pairwise voting process -# -# TODO: the /sse emission audit page needs a real SSE-driven UI. votes arrive -# incrementally during rank_commits(), and the client should show a live -# progress bar and per-vote results as they stream in. this requires a -# dedicated page that connects to /sse and updates the DOM on each event -# (council, comparing, vote, ranking, emission_complete). defer until we -# have playwright tests to cover it — the incremental rendering is fiddly. # =========================================================================== @app.get("/sse") @@ -1322,6 +1525,13 @@ async def sse_stream(request: Request): async def generate(): try: + yield _sse_event("audit", { + "id": AUDIT_SEQUENCE, + "timestamp_ms": int(time.time() * 1000), + "kind": "connection", + "message": f"Connected to epoch {current_epoch()[0]}", + **PROCESS_STATE, + }) yield exec_event(Three[Selector("#emission-status")][MORPH][ ["div#emission-status", f"Connected — epoch {current_epoch()[0]}"] ]) @@ -1334,10 +1544,15 @@ async def sse_stream(request: Request): except asyncio.TimeoutError: yield ": keepalive\n\n" finally: - SSE_CLIENTS.remove(queue) + if queue in SSE_CLIENTS: + SSE_CLIENTS.remove(queue) from starlette.responses import StreamingResponse - return StreamingResponse(generate(), media_type="text/event-stream") + return StreamingResponse( + generate(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) # =========================================================================== @@ -1359,6 +1574,7 @@ def _page(title: str, body: list) -> HTMLResponse: ["meta", {"charset": "utf-8"}], ["meta", {"name": "viewport", "content": "width=device-width, initial-scale=1"}], ["title", title], + ["style", RawContent(_WATCH_CSS)], ], ["body", body, @@ -1367,6 +1583,387 @@ def _page(title: str, body: list) -> HTMLResponse: ])) +_WATCH_CSS = """ +/* ================================================================ + ZIGGURAT — bevel-first dark theme + --spread (0→1) controls bevel depth. 0 = flat. 1 = full relief. + Light source: top-left. Shadow: bottom-right. + Platforms nest. Each level is raised. Nothing is rounded. + ================================================================ */ + +:root { + color-scheme: dark; + --spread: 1; + + --g0: #080808; + --g1: #131313; + --g2: #1c1c1c; + --g3: #252525; + --g4: #2e2e2e; + --g5: #383838; + + --hi: #5e5e5e; + --lo: #050505; + --bv: calc(var(--spread) * 4px + 1px); + --bv-lg: calc(var(--spread) * 6px + 2px); + + --signal: #f0f0f0; + --prose: #c2c2c2; + --ui: #888; + --meta: #4a4a4a; + --link: #8899ee; + --code-fg: #c8dda0; + + --font-prose: "Iowan Old Style", "Palatino Linotype", Palatino, "Book Antiqua", Georgia, serif; + --font-ui: system-ui, -apple-system, sans-serif; + --font-code: ui-monospace, "Cascadia Code", "SF Mono", Menlo, monospace; +} + +*, *::before, *::after { box-sizing: border-box; } +html, body { margin: 0; padding: 0; } + +body { + background: var(--g0); + color: var(--prose); + font-family: var(--font-ui); + font-size: 14px; + line-height: 1.6; + margin: 0 auto; + max-width: 560px; + min-height: 100vh; + padding: 0 16px 48px; +} +main { width: 100%; padding: 18px 0 48px; } + +h1, h2, h3 { + color: var(--signal); + font-size: 11px; + font-weight: bold; + letter-spacing: 0.12em; + margin: 14px 0 6px; + text-transform: uppercase; +} +a { color: var(--link); text-decoration: none; } +a:hover { color: var(--signal); } +.eyebrow { + background: var(--g2); + border: var(--bv) solid; + border-color: var(--hi) var(--lo) var(--lo) var(--hi); + color: var(--ui); + font-size: 11px; + letter-spacing: 0.12em; + padding: 4px 10px; + text-transform: uppercase; + width: fit-content; +} + +/* Every dashboard section is a raised platform. */ +.panel { + background: var(--g2); + border: var(--bv-lg) solid; + border-color: var(--hi) var(--lo) var(--lo) var(--hi); + margin: 8px 0; + padding: 10px; + width: 100%; +} +.status-row { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 8px; + justify-content: space-between; +} +#process-status { color: var(--signal); font-family: var(--font-code); font-weight: bold; } +.badge { + align-items: center; + background: var(--g3); + border: var(--bv) solid; + border-color: var(--hi) var(--lo) var(--lo) var(--hi); + color: var(--ui); + display: inline-flex; + font-size: 11px; + gap: 7px; + padding: 3px 8px; +} +.dot { background: var(--meta); height: 8px; width: 8px; } +.live .dot { background: #7acc7a; } +.warn .dot { background: #cc9955; } + +/* The progress track is inset; its signal is raised inside it. */ +.progress-shell { + background: var(--g1); + border: var(--bv-lg) solid; + border-color: var(--lo) var(--hi) var(--hi) var(--lo); + height: 58px; + margin: 14px 0 10px; + overflow: hidden; + position: relative; +} +#progress-fill { + background: var(--link); + border: var(--bv) solid; + border-color: var(--hi) var(--lo) var(--lo) var(--hi); + height: 100%; + transition: width .35s steps(8, end); + width: 0; +} +#progress-label { + color: var(--signal); + display: grid; + font-family: var(--font-code); + font-size: 18px; + font-weight: bold; + inset: 0; + place-items: center; + position: absolute; + text-shadow: 1px 1px var(--lo); +} + +.controls { align-items: center; display: flex; flex-wrap: wrap; gap: 8px; } +button { + background: var(--g5); + border: var(--bv) solid; + border-color: var(--hi) var(--lo) var(--lo) var(--hi); + color: var(--signal); + cursor: pointer; + font: inherit; + font-size: 12px; + padding: 4px 10px; +} +button:hover { background: #404040; } +button:active { + background: var(--g4); + border-color: var(--lo) var(--hi) var(--hi) var(--lo); + transform: translate(1px, 1px); +} +button:disabled { cursor: default; opacity: .4; } +.note { color: var(--meta); font-size: 11px; margin: 4px 0; } + +.feed-head { align-items: baseline; display: flex; justify-content: space-between; } +#audit-feed { + background: var(--g1); + border: var(--bv) solid; + border-color: var(--lo) var(--hi) var(--hi) var(--lo); + display: flex; + flex-direction: column; + gap: 5px; + margin-top: 8px; + padding: 6px; +} +.event { + background: var(--g3); + border: var(--bv) solid; + border-color: var(--hi) var(--lo) var(--lo) var(--hi); + display: grid; + gap: 6px; + grid-template-columns: 82px 88px 1fr; + padding: 5px 8px; +} +.event[data-kind="error"] { border-left-color: #cc5555; } +.event[data-kind="complete"], .event[data-kind="ranking"] { border-left-color: #7acc7a; } +.event[data-kind="vote"] { border-left-color: var(--link); } +.event time, .event-kind { color: var(--meta); font-family: var(--font-code); font-size: 10px; } +.event-kind { text-transform: uppercase; } +.event-message { color: var(--prose); font-family: var(--font-prose); } + +code { + background: var(--g1); + border: 2px solid; + border-color: var(--lo) var(--hi) var(--hi) var(--lo); + color: var(--code-fg); + font-family: var(--font-code); + font-size: 12px; + padding: 1px 4px; +} + +@media (max-width: 520px) { + .event { grid-template-columns: 72px 1fr; } + .event-message { grid-column: 1 / -1; } +} +""" + + +def _watch_initial_state() -> dict: + events = store.read() + feed = [] + for event_ in events[-40:]: + if isinstance(event_, GitDiscovery): + feed.append({ + "id": f"discovery-{event_.snapshot_id}", + "timestamp_ms": event_.timestamp_ms, + "kind": "discovery", + "message": ( + f"Epoch {event_.epoch}: observed {len(event_.observations)} commits; " + f"{len(event_.commits)} eligible" + ), + }) + elif isinstance(event_, Emission): + feed.append({ + "id": f"emission-{event_.epoch}", + "timestamp_ms": event_.timestamp_ms, + "kind": "complete", + "message": ( + f"Epoch {event_.epoch}: emitted {event_.total_emitted} SLG; " + f"ranking {event_.ranking}" + ), + }) + feed.extend(AUDIT_HISTORY) + return { + "process": dict(PROCESS_STATE), + "openrouter_configured": bool((OPENROUTER_API_KEY or "").strip()), + "epoch": current_epoch()[0], + "feed": feed[-200:], + } + + +_WATCH_JS = """ +const initial = __INITIAL__; +const feed = document.querySelector('#audit-feed'); +const processStatus = document.querySelector('#process-status'); +const connection = document.querySelector('#connection-status'); +const fill = document.querySelector('#progress-fill'); +const progressLabel = document.querySelector('#progress-label'); +const play = document.querySelector('#play'); +const pause = document.querySelector('#pause'); +const seen = new Set(); +let source = null; + +function setProgress(value) { + const n = Math.max(0, Math.min(100, Number(value ?? 0))); + fill.style.width = `${n}%`; + progressLabel.textContent = `${Math.round(n)}%`; + document.querySelector('.progress-shell').setAttribute('aria-valuenow', String(n)); +} + +function addEvent(event) { + const id = String(event.id); + if (seen.has(id)) return; + seen.add(id); + const row = document.createElement('div'); + row.className = 'event'; + row.dataset.kind = event.kind || 'event'; + const when = document.createElement('time'); + when.dateTime = new Date(event.timestamp_ms).toISOString(); + when.textContent = new Date(event.timestamp_ms).toLocaleTimeString(); + const kind = document.createElement('span'); + kind.className = 'event-kind'; + kind.textContent = event.kind || 'event'; + const message = document.createElement('span'); + message.className = 'event-message'; + message.textContent = event.message; + row.append(when, kind, message); + feed.prepend(row); + while (feed.children.length > 200) feed.lastElementChild.remove(); +} + +function applyState(event) { + processStatus.textContent = event.message || 'Waiting for the next epoch'; + setProgress(event.progress); + if (event.kind !== 'connection') addEvent(event); +} + +function connect() { + if (source) return; + source = new EventSource('/sse'); + connection.classList.remove('warn'); + connection.classList.add('live'); + connection.querySelector('span:last-child').textContent = 'connecting'; + play.disabled = true; + pause.disabled = false; + source.onopen = () => { + connection.querySelector('span:last-child').textContent = 'live'; + }; + source.addEventListener('audit', event => applyState(JSON.parse(event.data))); + source.onerror = () => { + connection.classList.remove('live'); + connection.classList.add('warn'); + connection.querySelector('span:last-child').textContent = 'reconnecting'; + }; +} + +function disconnect() { + if (source) source.close(); + source = null; + connection.classList.remove('live'); + connection.classList.add('warn'); + connection.querySelector('span:last-child').textContent = 'paused locally'; + play.disabled = false; + pause.disabled = true; +} + +play.addEventListener('click', connect); +pause.addEventListener('click', disconnect); +initial.feed.forEach(addEvent); +processStatus.textContent = initial.process.message; +setProgress(initial.process.progress); +connect(); +""" + + +@app.get("/watch") +async def watch(): + initial = json.dumps( + _watch_initial_state(), separators=(",", ":") + ).replace(" 0") - ;; 9. SSE connects and sends initial event + ;; 9. watch UI exposes progress, controls, readiness, and live SSE + (println "\nchecking /watch UI…") + (bind watch-html (slurp (str base-url "/watch"))) + (assert! (str/includes? watch-html "role=\"progressbar\"") + "watch page has progress bar") + (assert! (str/includes? watch-html "id=\"play\"") + "watch page has play control") + (assert! (str/includes? watch-html "id=\"pause\"") + "watch page has pause control") + (assert! (str/includes? watch-html "OpenRouter configured") + "watch page reports council readiness") + (bind status-resp (get-json base-url "/api/status")) + (assert! (true? (:openrouter_configured status-resp)) + "status API reports OpenRouter configuration") + + ;; 10. SSE connects and sends initial event (println "\nchecking /sse initial event…") (bind sse-events (read-sse-events (str base-url "/sse") 1 5000)) (assert! (= 1 (count sse-events)) "received 1 SSE event") (assert! (not (str/blank? (first sse-events))) "initial SSE event contains executable audit data") - ;; 10. POST /test/emit — full ranking pipeline hits mocks + ;; 11. POST /test/emit — full ranking pipeline hits mocks (println "\ntriggering /test/emit (epoch 1)…") (bind emit-resp (post-json! base-url "/test/emit")) (assert! (= "emission" (:type emit-resp)) "emit response type is emission") @@ -503,7 +518,7 @@ (bind rank-after (get-json base-url "/api/ranking")) (assert! (= 1 (:epoch rank-after)) "latest ranking is epoch 1") - ;; 11. kill and restart — prove replay determinism + ;; 12. kill and restart — prove replay determinism (println "\nkilling server for replay test…") (.destroyForcibly (:proc server)) (deref server) diff --git a/tests/test_git_discovery.py b/tests/test_git_discovery.py index 0dd31bc42a19bc8c59842dc61f193c595c474659..5a9e167e16ad2ffb25988af85820f6f19e8910cd 100644 --- a/tests/test_git_discovery.py +++ b/tests/test_git_discovery.py @@ -393,7 +393,9 @@ def test_empty_epoch_records_zero_emission_without_burning_pool( monkeypatch.setattr(c, "store", c.JsonlStore(discovery_config / "ledger.jsonl")) async def discover(_epoch, _boundary): - return SimpleNamespace(commits=[], snapshot_id="empty-snapshot") + return SimpleNamespace( + observations=[], commits=[], snapshot_id="empty-snapshot" + ) async def rank(_commits): return {}, [] @@ -412,7 +414,11 @@ def test_emission_distribution_sums_exactly_to_total( monkeypatch.setattr(c, "store", c.JsonlStore(discovery_config / "ledger.jsonl")) async def discover(_epoch, _boundary): - return SimpleNamespace(commits=[{"x": 1}], snapshot_id="ranked-snapshot") + return SimpleNamespace( + observations=[{"x": 1}], + commits=[{"x": 1}], + snapshot_id="ranked-snapshot", + ) async def rank(_commits): return { @@ -454,6 +460,7 @@ def test_any_council_failure_aborts_ranking(monkeypatch): monkeypatch.setattr(c, "fetch_top_models", models) monkeypatch.setattr(c, "llm_pairwise_compare", compare) + monkeypatch.setattr(c, "OPENROUTER_API_KEY", "test-key") commits = [ { "contributor": contributor, @@ -465,3 +472,54 @@ def test_any_council_failure_aborts_ranking(monkeypatch): ] with pytest.raises(RuntimeError, match="council model failed"): asyncio.run(c.rank_commits(commits)) + + +def test_contested_ranking_requires_openrouter_key(monkeypatch): + monkeypatch.setattr(c, "OPENROUTER_API_KEY", "") + commits = [ + { + "contributor": contributor, + "oid": "sha1:" + char * 40, + "message": contributor, + "patch": "patch", + } + for contributor, char in [("alice", "a"), ("bob", "b")] + ] + with pytest.raises(RuntimeError, match="OPENROUTER_API_KEY"): + asyncio.run(c.rank_commits(commits)) + + +def test_watch_page_has_live_controls_progress_and_key_warning( + discovery_config, monkeypatch +): + monkeypatch.setattr(c, "store", c.JsonlStore(discovery_config / "ledger.jsonl")) + monkeypatch.setattr(c, "OPENROUTER_API_KEY", "") + monkeypatch.setattr(c, "current_epoch", lambda: (3, 0, 1)) + response = asyncio.run(c.watch()) + html = response.body.decode() + assert 'role="progressbar"' in html + assert 'id="play"' in html + assert 'id="pause"' in html + assert "new EventSource('/sse')" in html + assert "OpenRouter key missing" in html + + +def test_audit_events_are_json_sse_and_update_process_state(monkeypatch): + clients = [] + history = [] + monkeypatch.setattr(c, "SSE_CLIENTS", clients) + monkeypatch.setattr(c, "AUDIT_HISTORY", history) + queue = asyncio.Queue() + clients.append(queue) + + async def emit(): + event = await c.broadcast_audit( + "progress", "halfway", progress=50, phase="ranking" + ) + return event, await queue.get() + + event, wire = asyncio.run(emit()) + assert event["progress"] == 50 + assert event["phase"] == "ranking" + assert wire.startswith("event: audit\ndata: {") + assert '"message":"halfway"' in wire