You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [08565bea] Tokenize prose refs for garden URL links (#147) * Tokenize prose refs for garden URL links Co-authored-by: tommy * Stop prose URLs at line boundaries Co-authored-by: tommy * Require braced DSL item bodies Co-authored-by: tommy --------- Co-authored-by: Cursor Agent Side A — unified diff (full patch): diff --git a/server/src/dsl.rs b/server/src/dsl.rs index 4203c2f59dd8825d7a91513c4023f3ccd37f1efc..b3c785674560a0b42a7f4c3aa13d80cd350f485b 100644 --- a/server/src/dsl.rs +++ b/server/src/dsl.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; -use rand::Rng; - /// Parsed DSL document. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Document { @@ -39,31 +37,57 @@ pub enum DslError { /// Matches the legacy Python parser behavior: /// - Supports toggle markers (open == close), e.g. ```...``` /// - Supports nested markers (open != close), e.g. { ... { ... } ... } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlockKind { + CodeFence, + DoubleBrace, + Brace, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct MaskedBlock { + kind: BlockKind, +} + #[derive(Debug, Default, Clone)] pub struct BlockMasker { pub replacements: HashMap, + blocks: HashMap, + next_id: u32, } impl BlockMasker { pub fn new() -> Self { Self { replacements: HashMap::new(), + blocks: HashMap::new(), + next_id: 0, } } - fn new_token(&mut self) -> String { - let mut rng = rand::thread_rng(); - let n: u32 = rng.gen(); - let token = format!("__BLOCK_{:08x}__", n); - // Extremely unlikely collision; if it happens, regenerate. - if self.replacements.contains_key(&token) { - return self.new_token(); + fn new_token(&mut self, haystack: &str) -> String { + loop { + let token = format!("__BLOCK_{:08x}__", self.next_id); + self.next_id = self.next_id.wrapping_add(1); + if !self.replacements.contains_key(&token) && !haystack.contains(&token) { + return token; + } } - token } /// Replace outermost balanced blocks with tokens. pub fn mask(&mut self, text: &str, open_marker: &str, close_marker: &str) -> String { + self.mask_kind(text, open_marker, close_marker, BlockKind::Brace) + } + + /// Replace outermost balanced blocks with typed deterministic tokens. + pub fn mask_kind( + &mut self, + text: &str, + open_marker: &str, + close_marker: &str, + kind: BlockKind, + ) -> String { if text.is_empty() { return text.to_string(); } @@ -97,9 +121,10 @@ impl BlockMasker { // Found end of outermost block let s = start_idx.max(0) as usize; let original_block = &text[s..i]; - let token = self.new_token(); + let token = self.new_token(text); self.replacements .insert(token.clone(), original_block.to_string()); + self.blocks.insert(token.clone(), MaskedBlock { kind }); result_parts.push(token); current_idx = i; } @@ -176,13 +201,22 @@ impl BlockMasker { } token.to_string() } + + pub fn block_kind(&self, token: &str) -> Option { + self.blocks.get(token).map(|b| b.kind) + } } fn mask_all(mut masker: BlockMasker, text: &str) -> (BlockMasker, String) { // Mask hierarchy: Code -> Double Brace -> Single Brace. - let t = masker.mask(text, "```", "```"); - let t = masker.mask(&t, "{{", "}}"); - let t = masker.mask(&t, "{", "}"); + let t = masker.mask_kind(text, "```", "```", BlockKind::CodeFence); + let t = masker.mask_kind(&t, "{{", "}}", BlockKind::DoubleBrace); + let t = masker.mask_kind(&t, "{", "}", BlockKind::Brace); + (masker, t) +} + +fn mask_code_fences(mut masker: BlockMasker, text: &str) -> (BlockMasker, String) { + let t = masker.mask_kind(text, "```", "```", BlockKind::CodeFence); (masker, t) } @@ -253,7 +287,34 @@ fn skip_ws(s: &str, mut i: usize) -> usize { i } -fn parse_item_name_at(s: &str, i: usize) -> Option<(String, usize)> { +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProseToken { + Text(String), + ItemRef(String), +} + +fn trim_prose_item_ref_end(s: &str, mut end: usize) -> usize { + while end > 0 { + let Some((idx, c)) = s[..end].char_indices().next_back() else { + break; + }; + if matches!( + c, + '.' | ',' | ';' | ':' | '!' | '?' | ')' | ']' | '}' | '"' | '\'' + ) { + end = idx; + } else { + break; + } + } + end +} + +fn parse_item_name_at_with_mode( + s: &str, + i: usize, + trim_trailing_punctuation: bool, +) -> Option<(String, usize)> { let bytes = s.as_bytes(); if i >= bytes.len() { return None; @@ -263,6 +324,9 @@ fn parse_item_name_at(s: &str, i: usize) -> Option<(String, usize)> { if s[i..].starts_with("https://") || s[i..].starts_with("http://") { let mut j = i; while j < bytes.len() { + if trim_trailing_punctuation && bytes[j] == b'\n' { + break; + } if bytes[j..].starts_with(b"__BLOCK_") || is_ws_byte(bytes[j]) { break; } @@ -271,6 +335,12 @@ fn parse_item_name_at(s: &str, i: usize) -> Option<(String, usize)> { if j <= i { return None; } + if trim_trailing_punctuation { + j = trim_prose_item_ref_end(s, j); + if j <= i { + return None; + } + } return Some((s[i..j].to_string(), j)); } @@ -296,6 +366,12 @@ fn parse_item_name_at(s: &str, i: usize) -> Option<(String, usize)> { if j <= i + 2 { return None; } + if trim_trailing_punctuation { + j = trim_prose_item_ref_end(s, j); + if j <= i + 2 { + return None; + } + } let raw = &s[i..j]; if !is_item_name(raw) { return None; @@ -336,6 +412,46 @@ fn parse_item_name_at(s: &str, i: usize) -> Option<(String, usize)> { Some((format!("~/{}", name), j)) } +fn parse_item_name_at(s: &str, i: usize) -> Option<(String, usize)> { + parse_item_name_at_with_mode(s, i, false) +} + +pub fn parse_prose_item_ref_at(s: &str, i: usize) -> Option<(String, usize)> { + parse_item_name_at_with_mode(s, i, true) +} + +pub fn tokenize_prose_item_refs(text: &str) -> Vec { + if text.is_empty() { + return Vec::new(); + } + let (masker, masked) = mask_code_fences(BlockMasker::new(), text); + let mut tokens = Vec::new(); + let mut text_start = 0usize; + let mut i = 0usize; + + while i < masked.len() { + if let Some((raw, end)) = parse_prose_item_ref_at(&masked, i) { + if text_start < i { + tokens.push(ProseToken::Text(masker.unmask(&masked[text_start..i]))); + } + tokens.push(ProseToken::ItemRef(masker.unmask(&raw))); + i = end; + text_start = i; + continue; + } + + let Some((_, c)) = masked[i..].char_indices().next() else { + break; + }; + i += c.len_utf8(); + } + + if text_start < masked.len() { + tokens.push(ProseToken::Text(masker.unmask(&masked[text_start..]))); + } + tokens +} + fn parse_block_token_at(s: &str, i: usize) -> Option<(String, usize)> { let bytes = s.as_bytes(); if i >= bytes.len() { @@ -401,6 +517,12 @@ fn parse_block_prefixed_statement( tail: &str, masker: &BlockMasker, ) -> Result { + if masker.block_kind(block_token) == Some(BlockKind::CodeFence) { + return Err(DslError::Parse( + "vote explanations must use `{ ... }`; code fences belong inside body blocks" + .to_string(), + )); + } // vote: block item_ref comparison item_ref let s = tail.trim_start(); if s.is_empty() { @@ -462,6 +584,11 @@ fn parse_item_definition_statement(stripped: &str, masker: &BlockMasker) -> Resu } if let Some((tok, end)) = parse_block_token_at(stripped, i) { + if masker.block_kind(&tok) == Some(BlockKind::CodeFence) { + return Err(DslError::Parse( + "item bodies must use `{ ... }`; code fences belong inside body blocks".to_string(), + )); + } let body = masker.extract_body(&tok); let tail = stripped[end..].trim(); if !tail.is_empty() { @@ -572,6 +699,10 @@ pub fn parse_full(text: &str) -> Result { if let Some((tok, end)) = parse_block_token_at(stripped, 0) { if stripped[end..].trim().is_empty() { + if masker.block_kind(&tok) == Some(BlockKind::CodeFence) { + prose_buffer.push(line); + continue; + } pending_block = Some(tok); continue; } @@ -619,6 +750,70 @@ mod tests { assert_eq!(roundtrip, input); } + #[test] + fn blockmasker_tokens_are_deterministic_and_typed() { + let input = "x ```code``` y {body}"; + let (masker, masked) = mask_all(BlockMasker::new(), input); + assert!(masked.contains("__BLOCK_00000000__")); + assert!(masked.contains("__BLOCK_00000001__")); + assert_eq!( + masker.block_kind("__BLOCK_00000000__"), + Some(BlockKind::CodeFence) + ); + assert_eq!( + masker.block_kind("__BLOCK_00000001__"), + Some(BlockKind::Brace) + ); + assert_eq!(masker.unmask(&masked), input); + } + + #[test] + fn prose_tokenizer_finds_tilde_dash_and_raw_url_refs() { + let tokens = + tokenize_prose_item_refs("see ~/a/b then -/example.com/x and https://Example.com/A/B."); + assert_eq!( + tokens, + vec![ + ProseToken::Text("see ".to_string()), + ProseToken::ItemRef("~/a/b".to_string()), + ProseToken::Text(" then ".to_string()), + ProseToken::ItemRef("-/example.com/x".to_string()), + ProseToken::Text(" and ".to_string()), + ProseToken::ItemRef("https://Example.com/A/B".to_string()), + ProseToken::Text(".".to_string()), + ] + ); + } + + #[test] + fn prose_tokenizer_stops_raw_urls_at_newlines() { + let tokens = tokenize_prose_item_refs("https://example.com/a/b.\n-/example.com/a/b"); + assert_eq!( + tokens, + vec![ + ProseToken::ItemRef("https://example.com/a/b".to_string()), + ProseToken::Text(".\n".to_string()), + ProseToken::ItemRef("-/example.com/a/b".to_string()), + ] + ); + } + + #[test] + fn prose_tokenizer_does_not_linkify_inside_code_fences() { + let tokens = tokenize_prose_item_refs( + "before ```json\n{\"url\":\"https://example.com\"}\n``` after ~/x", + ); + assert_eq!( + tokens, + vec![ + ProseToken::Text( + "before ```json\n{\"url\":\"https://example.com\"}\n``` after ".to_string() + ), + ProseToken::ItemRef("~/x".to_string()), + ] + ); + } + #[test] fn parse_item_with_body_strips_outer_braces() { let input = "~/rust { Systems language }"; @@ -633,8 +828,8 @@ mod tests { } #[test] - fn parse_item_with_fenced_json_body_preserves_braces() { - let input = "~/item/in/url ```json\n{\"test\": true}\n```"; + fn parse_item_with_braced_fenced_json_body_preserves_braces() { + let input = "~/item/in/url {\n```json\n{\"test\": true}\n```\n}"; let doc = parse_full(input).unwrap(); assert_eq!( doc.statements, @@ -645,6 +840,51 @@ mod tests { ); } + #[test] + fn parse_rejects_singleton_fenced_json_item_body() { + let input = "~/item/in/url ```json\n{\"test\": true}\n```"; + let err = parse_full(input).unwrap_err().to_string(); + assert!( + err.contains("item bodies must use"), + "unexpected error: {err}" + ); + } + + #[test] + fn parse_keeps_standalone_code_fence_as_prose() { + let input = "```json\n{\"test\": true}\n```"; + let doc = parse_full(input).unwrap(); + assert_eq!( + doc.statements, + vec![Stmt::Prose { + text: input.to_string(), + }] + ); + } + + #[test] + fn parse_rejects_code_fence_vote_explanation() { + let input = "```json\n{\"why\": true}\n```\n~/a 2:1 ~/b"; + let err = parse_full(input).unwrap_err().to_string(); + assert!( + err.contains("vote explanations must start"), + "unexpected error: {err}" + ); + } + + #[test] + fn parse_raw_url_item_with_braced_fenced_json_body() { + let input = "https://example.com/itembody/slug {\n```json\n{\"test\": true}\n```\n}"; + let doc = parse_full(input).unwrap(); + assert_eq!( + doc.statements, + vec![Stmt::Item { + title: "https://example.com/itembody/slug".to_string(), + body: Some("```json\n{\"test\": true}\n```".to_string()), + }] + ); + } + #[test] fn parse_vote_ratio_and_symbols() { let d1 = parse_full("{because}\n~/a 3:1 ~/b").unwrap(); diff --git a/server/src/external_resolver.rs b/server/src/external_resolver.rs index 8481f1fbff1c4265da182a225f539f97d6829002..04bc3de32b6df1aefe197d942db67de2cbdbde4f 100644 --- a/server/src/external_resolver.rs +++ b/server/src/external_resolver.rs @@ -256,7 +256,7 @@ fn children_to_dsl(children: &[ResolvedChild]) -> String { .filter(|s| !s.trim().is_empty()) .unwrap_or(child.title.as_str()); if body.trim_start().starts_with("```") { - out.push_str(&format!("{} {}\n\n", child.url, body.trim())); + out.push_str(&format!("{} {{\n{}\n}}\n\n", child.url, body.trim())); } else { out.push_str(&format!( "{} {{\n{}\n}}\n\n", @@ -378,7 +378,8 @@ mod tests { title: "#1 title".into(), body: Some("```json\n{\"test\": true}\n```".into()), }]); - assert!(dsl.contains("https://github.com/o/r/issues/1 ```json")); + assert!(dsl.contains("https://github.com/o/r/issues/1 {\n```json")); assert!(dsl.contains("{\"test\": true}")); + assert!(dsl.contains("```\n}\n")); } } diff --git a/server/src/html/breadcrumb_path.rs b/server/src/html/breadcrumb_path.rs index 3e949ce2cb70de3f024f46af773a93f3f8852260..e48612d7d3c4df52c161499a11865d4caca607aa 100644 --- a/server/src/html/breadcrumb_path.rs +++ b/server/src/html/breadcrumb_path.rs @@ -88,16 +88,12 @@ impl ExternalOntologyPath { .filter(|x| !x.is_empty()) .map(|x| x.to_string()) .collect(); - let segments = if segments == ["."] { - vec![] - } else { - segments - }; + let segments = if segments == ["."] { vec![] } else { segments }; Self { item, segments } } pub(super) fn is_root(&self) -> bool { - self.segments.len() <= 1 + self.segments.is_empty() } pub(super) fn segments(&self) -> &[String] { @@ -108,3 +104,28 @@ impl ExternalOntologyPath { self.item.as_str() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn external_root_and_host_paths_are_distinct() { + let root = ExternalOntologyPath::from_input(""); + assert!(root.is_root()); + assert!(root.segments().is_empty()); + + let host = ExternalOntologyPath::from_input("example.com"); + assert!(!host.is_root()); + assert_eq!(host.segments(), &["example.com".to_string()]); + } + + #[test] + fn external_path_keeps_each_url_segment_for_breadcrumbs() { + let path = ExternalOntologyPath::from_input("https://example.com/a/b"); + assert_eq!( + path.segments(), + &["example.com".to_string(), "a".to_string(), "b".to_string()] + ); + } +} diff --git a/server/src/html/mod.rs b/server/src/html/mod.rs index 8876a1a00a16ee3bf7a371f7a5ca27b5862b62cc..d5877eeb3632ed5d770f23a04cb4d1881209bc74 100644 --- a/server/src/html/mod.rs +++ b/server/src/html/mod.rs @@ -20,27 +20,30 @@ mod search; pub mod ui_action; use breadcrumb_path::{ExternalOntologyPath, OntologyPath}; -pub use auth::{auth_complete_page, auth_signed_in_fragment, choose_username_error_fragment, choose_username_page}; +pub use auth::{ + auth_complete_page, auth_signed_in_fragment, choose_username_error_fragment, + choose_username_page, +}; pub use editor::{editor_check, editor_page}; pub use forum::{ home, room_page, room_thread_post_view, room_thread_view, thread_feed_html, thread_feed_html_for_room, thread_feed_region_markup, thread_post_view, thread_view, ThreadNav, }; +pub use forum::user_profile_page; pub(crate) use forum::{ fragment_new_thread_slot, login_to_post_hint_markup, room_members_section_markup, thread_ui_collapse_redacted_post, thread_ui_expand_post_full, thread_ui_expand_redacted_post, user_can_post_room, user_can_view_room, }; +pub(crate) use garden::{encode_pin_cookie_value, vote_compare_post_success_js, GARDEN_PIN_COOKIE}; pub use garden::{ external_garden_index, external_ontology_path, garden_index, ontology_path, - room_external_garden_index, room_external_ontology_path, room_garden_index, - room_ontology_path, room_vote_compare_page, vote_compare_page, + room_external_garden_index, room_external_ontology_path, room_garden_index, room_ontology_path, + room_vote_compare_page, vote_compare_page, }; -pub(crate) use garden::{encode_pin_cookie_value, vote_compare_post_success_js, GARDEN_PIN_COOKIE}; pub use routing::RouteContext; pub use search::{search_page, search_results_fragment}; -pub use forum::user_profile_page; pub use ui_action::{parse_html_ui_from_form, HtmlUiAction, HtmlUiParseError, UI_RPC_FIELD}; /// Public profile URL path for a stored username (no `@`). @@ -107,8 +110,8 @@ pub struct ThemeForm { pub async fn post_theme(Form(form): Form) -> impl IntoResponse { let theme = normalize_theme(&form.theme); let next = sanitize_theme_next(form.next.as_deref()); - let loc = HeaderValue::try_from(next.as_str()) - .unwrap_or_else(|_| HeaderValue::from_static("/")); + let loc = + HeaderValue::try_from(next.as_str()).unwrap_or_else(|_| HeaderValue::from_static("/")); Response::builder() .status(StatusCode::SEE_OTHER) .header(header::LOCATION, loc) @@ -179,7 +182,9 @@ pub(crate) struct JsQueryBuilder { impl JsBuilder { pub(crate) fn new() -> Self { - Self { snippets: Vec::new() } + Self { + snippets: Vec::new(), + } } pub(crate) fn morph_selector(self, selector: &str, markup: Markup) -> Self { @@ -195,7 +200,12 @@ impl JsBuilder { self.qs(selector).morph_inner(markup) } - pub(crate) fn morph_expr(mut self, expr: &str, markup: Markup, morph_style: Option<&str>) -> Self { + pub(crate) fn morph_expr( + mut self, + expr: &str, + markup: Markup, + morph_style: Option<&str>, + ) -> Self { let html = js_string_literal(&markup.into_string()); let opts = morph_style .map(|style| format!(", {{morphStyle: {}}}", js_string_literal(style))) @@ -217,7 +227,11 @@ impl JsBuilder { self.qs(&format!("#{id}")) } - pub(crate) fn if_current_path_matches(mut self, path: &str, f: impl FnOnce(JsBuilder) -> JsBuilder) -> Self { + pub(crate) fn if_current_path_matches( + mut self, + path: &str, + f: impl FnOnce(JsBuilder) -> JsBuilder, + ) -> Self { let inner = f(JsBuilder::new()).build(); self.snippets.push(format!( "var __slugHere = window.location.pathname + window.location.search; var __slugPath = {path}; if (__slugHere === __slugPath || __slugHere.indexOf(__slugPath + '?') === 0) {{ {inner} }}", @@ -226,7 +240,11 @@ impl JsBuilder { self } - pub(crate) fn if_current_path_not_matches(mut self, path: &str, f: impl FnOnce(JsBuilder) -> JsBuilder) -> Self { + pub(crate) fn if_current_path_not_matches( + mut self, + path: &str, + f: impl FnOnce(JsBuilder) -> JsBuilder, + ) -> Self { let inner = f(JsBuilder::new()).build(); self.snippets.push(format!( "var __slugHere = window.location.pathname + window.location.search; var __slugPath = {path}; if (!(__slugHere === __slugPath || __slugHere.indexOf(__slugPath + '?') === 0)) {{ {inner} }}", @@ -301,7 +319,17 @@ pub(super) fn layout( garden_room_wire: Option<&str>, garden_path_prefix: Option<&str>, ) -> Markup { - layout_embed_controls(title, view, body, views, theme, theme_next, garden_room_wire, garden_path_prefix, true) + layout_embed_controls( + title, + view, + body, + views, + theme, + theme_next, + garden_room_wire, + garden_path_prefix, + true, + ) } /// Minimal document shell: no bottom controls, no garden HUD data attributes (`data-garden-room` / @@ -315,15 +343,7 @@ pub(super) fn layout_full_bleed_chromeless( theme_next: &str, ) -> Markup { layout_embed_controls( - title, - view, - body, - views, - theme, - theme_next, - None, - None, - false, + title, view, body, views, theme, theme_next, None, None, false, ) } @@ -545,7 +565,58 @@ fn item_body_title_snippet(body: &str) -> Option { Some(format!("{truncated}{ellipsis}")) } -/// Replace ~/path slugs in raw text with clickable links. +fn garden_href_for_item_ref( + raw_ref: &str, + garden_prefix: &str, +) -> Option<(crate::path_types::ItemId, String)> { + let key = slug_types::canonicalize_item(raw_ref); + let id = crate::path_types::ItemId::parse(&key)?; + let href = if let Some(tail) = id.tilde_tail() { + if tail.is_empty() { + garden_prefix.trim_end_matches('/').to_string() + } else { + format!("{}/{}", garden_prefix.trim_end_matches('/'), tail) + } + } else if id.as_str().starts_with("https://") || id.as_str().starts_with("http://") { + let display = id.display_path(); + let rest = display.strip_prefix("-/").unwrap_or(display.as_str()); + let ext_prefix = format!("{}-", garden_prefix.trim_end_matches('~')); + format!("{}/{}", ext_prefix, rest) + } else { + return None; + }; + Some((id, href)) +} + +fn push_item_ref_anchor( + out: &mut String, + raw_ref: &str, + garden_prefix: &str, + item_bodies: Option<&HashMap>, +) -> bool { + let Some((id, href)) = garden_href_for_item_ref(raw_ref, garden_prefix) else { + return false; + }; + out.push_str(r#"'); + out.push_str(&escape_html(raw_ref)); + out.push_str(""); + true +} + +/// Replace item refs in raw prose with clickable garden links. /// /// When `item_bodies` is set, matching ontology items get a `title` attribute with a truncated /// body preview for native browser tooltips (forum posts, item pages). @@ -554,53 +625,16 @@ pub(super) fn linkify_slugs_with_prefix( garden_prefix: &str, item_bodies: Option<&HashMap>, ) -> String { - let escaped = escape_html(raw); - let mut out = String::with_capacity(escaped.len() + 64); - let mut i = 0; - let s = escaped.as_str(); - while i < s.len() { - let rest = &s[i..]; - if let Some(after_tilde) = rest.strip_prefix("~/") { - let path_len = after_tilde - .chars() - .take_while(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '-' || *c == '/') - .map(|c| c.len_utf8()) - .sum::(); - if path_len > 0 { - let path = &after_tilde[..path_len]; - out.push_str(r#" out.push_str(&escape_html(&text)), + crate::dsl::ProseToken::ItemRef(raw_ref) => { + if !push_item_ref_anchor(&mut out, &raw_ref, garden_prefix, item_bodies) { + out.push_str(&escape_html(&raw_ref)); } - out.push('>'); - out.push_str("~/"); - out.push_str(path); - out.push_str(""); - i += 2 + path_len; - continue; } } - if let Some((j, c)) = rest.char_indices().next() { - out.push(c); - i += j + c.len_utf8(); - } else { - break; - } } out } @@ -638,7 +672,13 @@ fn spotify_embed_src(url: &str) -> Option { if !(host == "open.spotify.com" || host == "www.open.spotify.com") { return None; } - let path = tail.split('#').next().unwrap_or(tail).split('?').next().unwrap_or(tail); + let path = tail + .split('#') + .next() + .unwrap_or(tail) + .split('?') + .next() + .unwrap_or(tail); let mut segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); if segs.first().is_some_and(|s| s.starts_with("intl-")) { segs.remove(0); @@ -670,8 +710,16 @@ fn youtube_embed_src(url: &str) -> Option { let host = host.to_lowercase(); let video_id = if host == "youtu.be" || host == "www.youtu.be" { - clean_media_id(tail.split(['?', '#']).next().unwrap_or(tail).trim_matches('/')) - } else if matches!(host.as_str(), "youtube.com" | "www.youtube.com" | "m.youtube.com" | "music.youtube.com") { + clean_media_id( + tail.split(['?', '#']) + .next() + .unwrap_or(tail) + .trim_matches('/'), + ) + } else if matches!( + host.as_str(), + "youtube.com" | "www.youtube.com" | "m.youtube.com" | "music.youtube.com" + ) { let path = format!("/{}", tail.split('#').next().unwrap_or(tail)); if path.starts_with("/watch") { clean_media_id(&query_param(url, "v")?) @@ -745,10 +793,7 @@ pub(super) fn render_linkified_with_embeds_in_scope( /// CLI strings are embedded in a single-quoted JS literal; they must never need escaping. fn assert_cli_panel_cmd_js_single_quote_safe(s: &str) { assert!( - !s.contains('\\') - && !s.contains('\'') - && !s.contains('\n') - && !s.contains('\r'), + !s.contains('\\') && !s.contains('\'') && !s.contains('\n') && !s.contains('\r'), "cli_panel cmd must not contain `\\`, `'`, or newlines (got {s:?})" ); } @@ -805,15 +850,40 @@ mod linkify_title_tests { let mut bodies = HashMap::new(); let key = ItemId::parse(&slug_types::canonicalize_item("~/foo/bar")).unwrap(); bodies.insert(key, "Hello world\nline".to_string()); - let html = linkify_slugs_with_prefix( - "see ~/foo/bar ok", - "/r/x/~", - Some(&bodies), - ); + let html = linkify_slugs_with_prefix("see ~/foo/bar ok", "/r/x/~", Some(&bodies)); assert!(html.contains("title=\"Hello world line\"")); assert!(html.contains("href=\"/r/x/~/foo/bar\"")); } + #[test] + fn raw_url_links_to_public_external_garden_page() { + let html = linkify_slugs_with_prefix("see https://example.com/z.", "/~", None); + assert!(html + .contains(r#"https://example.com/z."#)); + } + + #[test] + fn dash_ref_links_to_room_external_garden_page_with_title() { + let mut bodies = HashMap::new(); + let key = ItemId::parse(&slug_types::canonicalize_item("-/example.com/z")).unwrap(); + bodies.insert(key, "External body\npreview".to_string()); + let html = + linkify_slugs_with_prefix("see -/example.com/z", "/r/9ab12cdroom/~", Some(&bodies)); + assert!(html.contains(r#"href="/r/9ab12cdroom/-/example.com/z""#)); + assert!(html.contains(r#"title="External body preview""#)); + } + + #[test] + fn code_fence_urls_are_not_linkified() { + let html = linkify_slugs_with_prefix( + "```json\n{\"url\":\"https://example.com/z\"}\n```\nthen https://example.com/a", + "/~", + None, + ); + assert!(!html.contains(r#"href="/-/example.com/z""#)); + assert!(html.contains(r#"href="/-/example.com/a""#)); + } + #[test] fn no_title_when_body_missing_or_empty() { let html = linkify_slugs_with_prefix("x ~/a/b y", "/~", Some(&HashMap::new())); Side B — contributor: tommy-mor Side B — commit message: [15e1037a] url stuff Side B — unified diff (full patch): diff --git a/server/src/url_rules/graph.rs b/server/src/url_rules/graph.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7ac0f9a551a1727cb2f9294778c283b9885b147 --- /dev/null +++ b/server/src/url_rules/graph.rs @@ -0,0 +1,831 @@ +//! Semantic URL graph: DFA traversal on host + path, query in context, generic fallback. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use url::Url; + +use super::graph_builder::GraphBuilder; +use super::parse::{normalize_match_host, strip_tracking_query, UrlParts}; + +#[derive(Debug, Clone, Default)] +pub struct Context { + pub vars: HashMap, + pub query: HashMap, +} + +pub type CanonicalFn = fn(&Context) -> Option; + +#[derive(Clone, Copy)] +pub enum EdgePattern { + Literal(&'static str), + Variable(&'static str), + /// Absorb any trailing segment without leaving this node (e.g. post title slug). + AbsorbAny, + /// Absorb segment when `cond(seg)` (e.g. subreddit listing suffix). + AbsorbIf(fn(&str) -> bool), +} + +pub struct Edge { + pub pattern: EdgePattern, + pub target: &'static str, +} + +pub struct Node { + pub edges: Vec, + pub canonical: CanonicalFn, + pub parent: Option<&'static str>, +} + +impl Node { + pub(crate) fn empty() -> Self { + Self { + edges: Vec::new(), + canonical: |_| None, + parent: None, + } + } +} + +pub struct Graph { + pub nodes: HashMap<&'static str, Node>, +} + +static GRAPH: OnceLock = OnceLock::new(); + +pub fn graph() -> &'static Graph { + GRAPH.get_or_init(build_graph) +} + +impl Graph { + pub fn resolve_canonical(&self, parts: &UrlParts) -> Option { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + + if let Some(node_id) = self.traverse(parts, &mut ctx) { + if let Some(canon) = (self.nodes.get(node_id)?.canonical)(&ctx) { + return Some(canon); + } + } + Some(generic_canonical(parts)) + } + + pub fn breadcrumbs(&self, parts: &UrlParts) -> Vec { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + + if let Some(mut node_id) = self.traverse(parts, &mut ctx) { + let mut paths = Vec::new(); + loop { + let node = match self.nodes.get(node_id) { + Some(n) => n, + None => break, + }; + if let Some(url) = (node.canonical)(&ctx) { + if paths.last() != Some(&url) { + paths.push(url); + } + } + match node.parent { + Some(p) => node_id = p, + None => break, + } + } + paths.reverse(); + if !paths.is_empty() { + return paths; + } + } + generic_breadcrumbs(parts) + } + + fn traverse(&self, parts: &UrlParts, ctx: &mut Context) -> Option<&'static str> { + let host = parts.match_host(); + let mut node_id = match host.as_str() { + "reddit.com" => "reddit_root", + "youtube.com" => "youtube_root", + "youtu.be" => "youtu_be_entry", + _ => return None, + }; + + let segs: Vec<&str> = parts.path_segments.iter().map(String::as_str).collect(); + let mut i = 0; + while i < segs.len() { + let seg = segs[i]; + match self.follow_edge(node_id, seg, ctx) { + Ok(next) => { + node_id = next; + i += 1; + } + Err(()) => { + if self.try_absorb(node_id, seg) { + i += 1; + continue; + } + return None; + } + } + } + Some(node_id) + } + + fn follow_edge( + &self, + node_id: &'static str, + seg: &str, + ctx: &mut Context, + ) -> Result<&'static str, ()> { + let node = self.nodes.get(node_id).ok_or(())?; + for edge in &node.edges { + match edge.pattern { + EdgePattern::Literal(lit) if lit == seg => return Ok(edge.target), + EdgePattern::Variable(name) => { + ctx.vars.insert(name.to_string(), seg.to_string()); + return Ok(edge.target); + } + EdgePattern::AbsorbAny + | EdgePattern::AbsorbIf(_) + | EdgePattern::Literal(_) + | EdgePattern::Variable(_) => {} + } + } + Err(()) + } + + fn try_absorb(&self, node_id: &'static str, seg: &str) -> bool { + let node = match self.nodes.get(node_id) { + Some(n) => n, + None => return false, + }; + for edge in &node.edges { + match edge.pattern { + EdgePattern::AbsorbAny => return true, + EdgePattern::AbsorbIf(cond) if cond(seg) => return true, + EdgePattern::AbsorbIf(_) | EdgePattern::Literal(_) | EdgePattern::Variable(_) => {} + } + } + false + } + + /// Test hook: terminal graph node and captured context after traversal. + #[cfg(test)] + pub fn traverse_terminal(&self, parts: &UrlParts) -> Option<(&'static str, Context)> { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + let node = self.traverse(parts, &mut ctx)?; + Some((node, ctx)) + } +} + +fn is_reddit_listing_suffix(seg: &str) -> bool { + matches!(seg, "hot" | "top" | "new" | "rising" | "controversial") +} + +/// Percent-encode a path or query fragment so `&`, `?`, etc. cannot break URL structure. +fn enc(s: &str) -> String { + urlencoding::encode(s).into_owned() +} + +// --- Canonical formatters --- + +fn canon_reddit_root(_: &Context) -> Option { + Some("https://reddit.com".to_string()) +} + +fn canon_reddit_r_hub(_: &Context) -> Option { + Some("https://reddit.com/r".to_string()) +} + +fn canon_reddit_subreddit(ctx: &Context) -> Option { + let sub = ctx.vars.get("subreddit")?; + Some(format!( + "https://reddit.com/r/{}", + enc(&sub.to_ascii_lowercase()) + )) +} + +fn canon_reddit_post(ctx: &Context) -> Option { + let sub = ctx.vars.get("subreddit")?.to_ascii_lowercase(); + let id = ctx.vars.get("post_id")?; + Some(format!( + "https://reddit.com/r/{}/comments/{}", + enc(&sub), + enc(id) + )) +} + +fn canon_youtube_root(_: &Context) -> Option { + Some("https://youtube.com".to_string()) +} + +fn canon_youtube_watch(ctx: &Context) -> Option { + let v = ctx + .query + .get("v") + .or_else(|| ctx.vars.get("video_id"))?; + Some(format!("https://youtube.com/watch?v={}", enc(v))) +} + +fn canon_youtu_be(ctx: &Context) -> Option { + let v = ctx.vars.get("vid_id")?; + Some(format!("https://youtube.com/watch?v={}", enc(v))) +} + +pub fn build_graph() -> Graph { + GraphBuilder::new() + .node("reddit_root") + .canonical(canon_reddit_root) + .edge(EdgePattern::Literal("r"), "reddit_r_hub") + .node("reddit_r_hub") + .parent("reddit_root") + .canonical(canon_reddit_r_hub) + .edge(EdgePattern::Variable("subreddit"), "reddit_subreddit") + .node("reddit_subreddit") + .parent("reddit_r_hub") + .canonical(canon_reddit_subreddit) + .edge( + EdgePattern::AbsorbIf(is_reddit_listing_suffix), + "reddit_subreddit", + ) + .edge(EdgePattern::Literal("comments"), "reddit_comments_gate") + .node("reddit_comments_gate") + .parent("reddit_subreddit") + .canonical(canon_reddit_subreddit) + .edge(EdgePattern::Variable("post_id"), "reddit_post") + .node("reddit_post") + .parent("reddit_subreddit") + .canonical(canon_reddit_post) + .edge(EdgePattern::AbsorbAny, "reddit_post") + .node("youtube_root") + .canonical(canon_youtube_root) + .edge(EdgePattern::Literal("watch"), "youtube_watch") + .edge(EdgePattern::Literal("shorts"), "youtube_shorts_gate") + .node("youtube_watch") + .parent("youtube_root") + .canonical(canon_youtube_watch) + .node("youtube_shorts_gate") + .parent("youtube_root") + .canonical(canon_youtube_root) + .edge(EdgePattern::Variable("video_id"), "youtube_watch") + .node("youtu_be_entry") + .canonical(canon_youtube_root) + .edge(EdgePattern::Variable("vid_id"), "youtu_be_video") + .node("youtu_be_video") + .parent("youtube_root") + .canonical(canon_youtu_be) + .build() +} + +// --- Generic internet fallback --- + +pub fn generic_canonical(parts: &UrlParts) -> String { + let host = normalize_match_host(&parts.host); + let path_segments: Vec = parts.path_segments.clone(); + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + + let mut url = if path_segments.is_empty() { + Url::parse(&format!("https://{host}")) + .unwrap_or_else(|_| Url::parse("https://invalid").unwrap()) + } else { + let path = format!("/{}", path_segments.join("/")); + Url::parse(&format!("https://{host}{path}")) + .unwrap_or_else(|_| Url::parse("https://invalid").unwrap()) + }; + + if !query.is_empty() { + let mut pairs: Vec<_> = query.iter().collect(); + pairs.sort_by(|a, b| a.0.cmp(b.0)); + url.query_pairs_mut().clear(); + for (k, v) in pairs { + url.query_pairs_mut().append_pair(k, v); + } + } + + let mut s = url.to_string(); + if path_segments.is_empty() { + s = s.trim_end_matches('/').to_string(); + } + s +} + +pub fn generic_breadcrumbs(parts: &UrlParts) -> Vec { + let host = normalize_match_host(&parts.host); + let n = parts.path_segments.len(); + let mut out = Vec::new(); + + let base = generic_canonical(&UrlParts { + scheme: "https".to_string(), + host: host.clone(), + path_segments: vec![], + query: HashMap::new(), + }); + out.push(base); + + for i in 0..n { + let segs: Vec = parts.path_segments[..=i].to_vec(); + let url = generic_canonical(&UrlParts { + scheme: "https".to_string(), + host: host.clone(), + path_segments: segs, + query: HashMap::new(), + }); + if out.last() != Some(&url) { + out.push(url); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::url_rules::parse::test_parts; + + fn g() -> &'static Graph { + graph() + } + + fn canon(parts: &UrlParts) -> String { + g().resolve_canonical(parts).unwrap() + } + + fn crumbs(parts: &UrlParts) -> Vec { + g().breadcrumbs(parts) + } + + fn terminal(parts: &UrlParts) -> Option<&'static str> { + g().traverse_terminal(parts).map(|(n, _)| n) + } + + fn vars(parts: &UrlParts) -> HashMap { + g().traverse_terminal(parts) + .map(|(_, c)| c.vars) + .unwrap_or_default() + } + + #[test] + fn youtu_be_malicious_segment_encoded_not_injected() { + let p = test_parts("youtu.be", &["abc&t=1"], &[]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=abc%26t%3D1"); + assert!(!canon(&p).contains("abc&t=1")); + } + + #[test] + fn youtube_query_v_encoded() { + let p = test_parts("youtube.com", &["watch"], &[("v", "a&b=c")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=a%26b%3Dc"); + } + + #[test] + fn absorb_patterns_live_on_edges_not_in_engine() { + let g = build_graph(); + let sub = g.nodes.get("reddit_subreddit").unwrap(); + assert!(sub + .edges + .iter() + .any(|e| matches!(e.pattern, EdgePattern::AbsorbIf(_)))); + let post = g.nodes.get("reddit_post").unwrap(); + assert!(post + .edges + .iter() + .any(|e| matches!(e.pattern, EdgePattern::AbsorbAny))); + } + + #[test] + fn builder_rejects_missing_parent() { + let result = std::panic::catch_unwind(|| { + GraphBuilder::new() + .node("orphan") + .parent("nonexistent_parent") + .build(); + }); + assert!(result.is_err()); + } + + #[test] + fn generic_canonical_forces_https_and_strips_www() { + let p = test_parts("www.example.com", &["blog", "post"], &[]); + assert_eq!(canon(&p), "https://example.com/blog/post"); + } + + #[test] + fn generic_canonical_sorts_query_keys() { + let p = test_parts("example.com", &["search"], &[("q", "rust"), ("page", "2")]); + assert_eq!(canon(&p), "https://example.com/search?page=2&q=rust"); + } + + #[test] + fn generic_canonical_strips_tracking_from_query() { + let p = test_parts( + "news.ycombinator.com", + &["item"], + &[("id", "1"), ("utm_medium", "social")], + ); + assert_eq!(canon(&p), "https://news.ycombinator.com/item?id=1"); + } + + #[test] + fn generic_breadcrumbs_cumulative_path() { + let p = test_parts("paulgraham.com", &["articles", "lisp.html"], &[]); + assert_eq!( + crumbs(&p), + vec![ + "https://paulgraham.com", + "https://paulgraham.com/articles", + "https://paulgraham.com/articles/lisp.html" + ] + ); + } + + #[test] + fn generic_breadcrumbs_domain_only() { + let p = test_parts("example.com", &[], &[]); + assert_eq!(crumbs(&p), vec!["https://example.com"]); + } + + #[test] + fn unknown_host_uses_generic_not_graph() { + let p = test_parts("hackernews.com", &["item", "123"], &[]); + assert_eq!(terminal(&p), None); + assert_eq!(canon(&p), "https://hackernews.com/item/123"); + } + + #[test] + fn traverse_captures_subreddit_variable() { + let p = test_parts("reddit.com", &["r", "Rust"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(vars(&p).get("subreddit").map(String::as_str), Some("Rust")); + } + + #[test] + fn traverse_captures_post_id() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "abc123"], &[]); + assert_eq!(terminal(&p), Some("reddit_post")); + assert_eq!(vars(&p).get("post_id").map(String::as_str), Some("abc123")); + } + + #[test] + fn traverse_absorbs_listing_suffix_stays_on_subreddit() { + let p = test_parts("reddit.com", &["r", "rust", "hot"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn traverse_absorbs_all_listing_suffixes() { + for suffix in ["hot", "top", "new", "rising", "controversial"] { + let p = test_parts("reddit.com", &["r", "test", suffix], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit"), "suffix {suffix}"); + assert_eq!(canon(&p), "https://reddit.com/r/test", "suffix {suffix}"); + } + } + + #[test] + fn traverse_absorbs_post_title_slug() { + let p = test_parts( + "reddit.com", + &["r", "rust", "comments", "aaa", "my_great_post_title"], + &[], + ); + assert_eq!(terminal(&p), Some("reddit_post")); + assert_eq!(canon(&p), "https://reddit.com/r/rust/comments/aaa"); + } + + #[test] + fn traverse_unknown_segment_falls_back_to_generic() { + let p = test_parts("reddit.com", &["r", "rust", "wiki", "faq"], &[]); + assert_eq!(terminal(&p), None); + assert_eq!(canon(&p), "https://reddit.com/r/rust/wiki/faq"); + } + + #[test] + fn traverse_youtube_watch_requires_v_in_query() { + let p = test_parts("youtube.com", &["watch"], &[("v", "xyz")]); + assert_eq!(terminal(&p), Some("youtube_watch")); + } + + #[test] + fn traverse_youtu_be_captures_vid_id() { + let p = test_parts("youtu.be", &["dQw4w9WgXcQ"], &[]); + assert_eq!(terminal(&p), Some("youtu_be_video")); + assert_eq!(vars(&p).get("vid_id").map(String::as_str), Some("dQw4w9WgXcQ")); + } + + #[test] + fn traverse_shorts_sets_video_id_var() { + let p = test_parts("youtube.com", &["shorts", "abc99"], &[]); + assert_eq!(terminal(&p), Some("youtube_watch")); + assert_eq!(vars(&p).get("video_id").map(String::as_str), Some("abc99")); + } + + #[test] + fn reddit_domain_canonical() { + let p = test_parts("reddit.com", &[], &[]); + assert_eq!(canon(&p), "https://reddit.com"); + } + + #[test] + fn reddit_r_hub_canonical() { + let p = test_parts("reddit.com", &["r"], &[]); + assert_eq!(terminal(&p), Some("reddit_r_hub")); + assert_eq!(canon(&p), "https://reddit.com/r"); + } + + #[test] + fn reddit_subreddit_lowercases_name() { + let p = test_parts("reddit.com", &["r", "AmITheAsshole"], &[]); + assert_eq!(canon(&p), "https://reddit.com/r/amitheasshole"); + } + + #[test] + fn reddit_host_aliases_old_new_www() { + for host in ["old.reddit.com", "new.reddit.com", "www.reddit.com"] { + let p = test_parts(host, &["r", "rust"], &[]); + assert_eq!(canon(&p), "https://reddit.com/r/rust", "host {host}"); + } + } + + #[test] + fn reddit_post_strips_slug_and_query() { + let p = test_parts( + "old.reddit.com", + &["r", "Rust", "comments", "1abc", "title_slug_here"], + &[("sort", "new")], + ); + assert_eq!(canon(&p), "https://reddit.com/r/rust/comments/1abc"); + } + + #[test] + fn reddit_post_multiple_slugs_absorbed() { + let p = test_parts( + "reddit.com", + &["r", "x", "comments", "id1", "slug1", "extra"], + &[], + ); + assert_eq!(canon(&p), "https://reddit.com/r/x/comments/id1"); + } + + #[test] + fn reddit_listing_with_query_only() { + let p = test_parts("www.reddit.com", &["r", "programming"], &[("sort", "top")]); + assert_eq!(canon(&p), "https://reddit.com/r/programming"); + } + + #[test] + fn reddit_subreddit_breadcrumbs_include_r_hub() { + let p = test_parts("reddit.com", &["r", "movies"], &[]); + assert_eq!( + crumbs(&p), + vec![ + "https://reddit.com", + "https://reddit.com/r", + "https://reddit.com/r/movies" + ] + ); + } + + #[test] + fn reddit_post_breadcrumbs_skip_comments_node() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "1trnvdl"], &[]); + let c = crumbs(&p); + assert!(!c.iter().any(|u| u.ends_with("/comments"))); + assert_eq!( + c.last().map(String::as_str), + Some("https://reddit.com/r/aww/comments/1trnvdl") + ); + assert!(c.contains(&"https://reddit.com/r/aww".to_string())); + } + + #[test] + fn reddit_post_parent_is_subreddit_not_comments() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "1trnvdl"], &[]); + let c = crumbs(&p); + let parent = c.get(c.len() - 2).unwrap(); + assert_eq!(parent, "https://reddit.com/r/aww"); + } + + #[test] + fn reddit_domain_parent_is_none_in_breadcrumb_chain() { + let p = test_parts("reddit.com", &[], &[]); + assert_eq!(crumbs(&p), vec!["https://reddit.com"]); + } + + #[test] + fn youtube_watch_canonical_uses_v_only() { + let p = test_parts("youtube.com", &["watch"], &[("v", "abc"), ("t", "99")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=abc"); + } + + #[test] + fn youtube_query_order_independent() { + let a = test_parts("youtube.com", &["watch"], &[("v", "abc"), ("t", "4")]); + let b = test_parts("youtube.com", &["watch"], &[("t", "4"), ("v", "abc")]); + assert_eq!(canon(&a), canon(&b)); + } + + #[test] + fn youtube_host_aliases() { + for host in ["www.youtube.com", "m.youtube.com"] { + let p = test_parts(host, &["watch"], &[("v", "x")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=x", "host {host}"); + } + } + + #[test] + fn youtube_shorts_canonical_matches_watch() { + let shorts = test_parts("youtube.com", &["shorts", "vid123"], &[]); + let watch = test_parts("youtube.com", &["watch"], &[("v", "vid123")]); + assert_eq!(canon(&shorts), canon(&watch)); + assert_eq!(canon(&shorts), "https://youtube.com/watch?v=vid123"); + } + + #[test] + fn youtu_be_matches_youtube_watch() { + let be = test_parts("youtu.be", &["dQw4w9WgXcQ"], &[]); + let watch = test_parts("youtube.com", &["watch"], &[("v", "dQw4w9WgXcQ")]); + assert_eq!(canon(&be), canon(&watch)); + } + + #[test] + fn youtube_breadcrumbs_domain_then_watch() { + let p = test_parts("youtube.com", &["watch"], &[("v", "abc")]); + assert_eq!( + crumbs(&p), + vec!["https://youtube.com", "https://youtube.com/watch?v=abc"] + ); + } + + #[test] + fn youtu_be_breadcrumbs_include_youtube_domain() { + let p = test_parts("youtu.be", &["abc"], &[]); + let c = crumbs(&p); + assert_eq!(c.first().map(String::as_str), Some("https://youtube.com")); + assert_eq!( + c.last().map(String::as_str), + Some("https://youtube.com/watch?v=abc") + ); + } + + #[test] + fn parsed_urls_match_hand_built_parts() { + let raw = "https://www.reddit.com/r/rust/comments/aaa/title/?utm=x"; + let parsed = UrlParts::parse(raw).unwrap(); + let hand = test_parts( + "www.reddit.com", + &["r", "rust", "comments", "aaa", "title"], + &[("utm", "x")], + ); + assert_eq!(canon(&parsed), canon(&hand)); + } + + #[test] + fn equivalence_cluster_youtube_formats() { + let urls = [ + "https://youtu.be/abc123", + "https://www.youtube.com/watch?v=abc123", + "https://youtube.com/watch?v=abc123&t=1", + "https://m.youtube.com/watch?t=1&v=abc123", + ]; + let canonical: Vec<_> = urls + .iter() + .map(|u| canon(&UrlParts::parse(u).unwrap())) + .collect(); + assert!(canonical.iter().all(|c| *c == "https://youtube.com/watch?v=abc123")); + } + + #[test] + fn equivalence_cluster_reddit_post_formats() { + let urls = [ + "https://old.reddit.com/r/Rust/comments/aaa/slug/", + "reddit.com/r/rust/comments/aaa/other_slug", + "https://reddit.com/r/RUST/comments/aaa", + ]; + let canonical: Vec<_> = urls + .iter() + .map(|u| canon(&UrlParts::parse(u).unwrap())) + .collect(); + assert!( + canonical + .iter() + .all(|c| *c == "https://reddit.com/r/rust/comments/aaa") + ); + } + + #[test] + fn graph_nodes_all_have_valid_parent_links() { + let g = build_graph(); + for (id, node) in &g.nodes { + if let Some(parent) = node.parent { + assert!(g.nodes.contains_key(parent), "node {id} parent {parent}"); + } + } + } + + #[test] + fn graph_terminal_canonical_always_succeeds_for_reddit_paths() { + let cases: &[(&[&str], &str)] = &[ + (&["r", "rust"], "https://reddit.com/r/rust"), + ( + &["r", "rust", "comments", "x"], + "https://reddit.com/r/rust/comments/x", + ), + ]; + for (segs, want) in cases { + let p = test_parts("reddit.com", segs, &[]); + assert_eq!(canon(&p), *want); + } + } + + #[test] + fn breadcrumb_parent_walk_matches_parent_url_semantics() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "id1"], &[]); + let c = crumbs(&p); + assert_eq!(c.len(), 4); + assert_eq!( + c.get(c.len() - 2).map(String::as_str), + Some("https://reddit.com/r/aww") + ); + } + + #[test] + fn youtube_watch_without_v_falls_back_to_generic() { + let p = test_parts("youtube.com", &["watch"], &[]); + assert_eq!(terminal(&p), Some("youtube_watch")); + assert_eq!(canon(&p), "https://youtube.com/watch"); + } + + #[test] + fn reddit_only_comments_path_stops_at_gate() { + let p = test_parts("reddit.com", &["r", "rust", "comments"], &[]); + assert_eq!(terminal(&p), Some("reddit_comments_gate")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn generic_deep_path_many_segments() { + let segs: Vec<&str> = (0..10) + .map(|i| match i { + 0 => "a", + 1 => "b", + 2 => "c", + 3 => "d", + 4 => "e", + 5 => "f", + 6 => "g", + 7 => "h", + 8 => "i", + _ => "j", + }) + .collect(); + let p = test_parts("site.com", &segs, &[]); + assert_eq!(crumbs(&p).len(), 11); + } + + #[test] + fn traverse_literal_r_required_for_subreddit() { + let p = test_parts("reddit.com", &["rust"], &[]); + assert_eq!(terminal(&p), None); + } + + #[test] + fn http_scheme_upgraded_via_generic_fallback_host() { + let parsed = UrlParts::parse("http://example.com/page").unwrap(); + assert_eq!(canon(&parsed), "https://example.com/page"); + } + + #[test] + fn each_graph_node_canonical_is_invokable() { + let g = build_graph(); + let empty = Context::default(); + for (id, node) in &g.nodes { + let _ = (node.canonical)(&empty); + let _ = id; + } + } + + #[test] + fn reddit_double_listing_suffix_both_absorbed() { + let p = test_parts("reddit.com", &["r", "rust", "hot", "new"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn youtu_be_empty_path_stays_at_entry() { + let p = test_parts("youtu.be", &[], &[]); + assert_eq!(terminal(&p), Some("youtu_be_entry")); + } +} diff --git a/server/src/url_rules/graph_builder.rs b/server/src/url_rules/graph_builder.rs new file mode 100644 index 0000000000000000000000000000000000000000..243204ad5ca514fff459057956080cacb5bf38e2 --- /dev/null +++ b/server/src/url_rules/graph_builder.rs @@ -0,0 +1,73 @@ +//! Declarative construction of the URL graph with build-time link validation. + +use std::collections::HashMap; + +use super::graph::{CanonicalFn, Edge, EdgePattern, Graph, Node}; + +pub struct GraphBuilder { + nodes: HashMap<&'static str, Node>, + current: Option<&'static str>, +} + +impl GraphBuilder { + pub fn new() -> Self { + Self { + nodes: HashMap::new(), + current: None, + } + } + + pub fn node(mut self, id: &'static str) -> Self { + self.nodes.entry(id).or_insert_with(Node::empty); + self.current = Some(id); + self + } + + pub fn canonical(mut self, f: CanonicalFn) -> Self { + let id = self.current.expect("canonical() without node()"); + self.nodes.get_mut(id).expect("node missing").canonical = f; + self + } + + pub fn parent(mut self, parent_id: &'static str) -> Self { + let id = self.current.expect("parent() without node()"); + self.nodes.get_mut(id).expect("node missing").parent = Some(parent_id); + self + } + + pub fn edge(mut self, pattern: EdgePattern, target: &'static str) -> Self { + let id = self.current.expect("edge() without node()"); + self.nodes + .get_mut(id) + .expect("node missing") + .edges + .push(Edge { pattern, target }); + self + } + + pub fn build(self) -> Graph { + for (id, node) in &self.nodes { + if let Some(parent) = node.parent { + assert!( + self.nodes.contains_key(parent), + "node {id}: parent {parent} does not exist" + ); + } + for edge in &node.edges { + if !matches!( + edge.pattern, + EdgePattern::AbsorbAny | EdgePattern::AbsorbIf(_) + ) { + assert!( + self.nodes.contains_key(edge.target), + "node {id}: edge target {} does not exist", + edge.target + ); + } + } + } + Graph { + nodes: self.nodes, + } + } +} diff --git a/server/src/url_rules/mod.rs b/server/src/url_rules/mod.rs index 9e1445346ce77a49dd6a7e7713bf9c57aef353cc..ba4ac662acc7bd4d50ee34613eb7eb6fccbfb17b 100644 --- a/server/src/url_rules/mod.rs +++ b/server/src/url_rules/mod.rs @@ -1,6 +1,7 @@ //! URL canonicalization and hierarchy via a semantic graph (DFA + generic fallback). mod graph; +mod graph_builder; mod parse; mod registry; diff --git a/server/src/url_rules/parse.rs b/server/src/url_rules/parse.rs new file mode 100644 index 0000000000000000000000000000000000000000..19d0c82feb718817168b8b445fcbfa39cf6aa6ba --- /dev/null +++ b/server/src/url_rules/parse.rs @@ -0,0 +1,169 @@ +//! Parse raw strings into host, path segments, and query (order-independent). + +use std::collections::HashMap; + +use url::Url; + +#[derive(Debug, Clone)] +pub struct UrlParts { + pub scheme: String, + pub host: String, + pub path_segments: Vec, + pub query: HashMap, +} + +impl UrlParts { + pub fn parse(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + let with_scheme = if trimmed.contains("://") { + trimmed.to_string() + } else if trimmed.starts_with("r/") || trimmed.starts_with("/r/") { + let rest = trimmed.trim_start_matches('/').trim_start_matches("r/"); + format!("https://reddit.com/r/{rest}") + } else if trimmed.contains('.') && !trimmed.starts_with('/') { + format!("https://{trimmed}") + } else { + trimmed.to_string() + }; + + let url = Url::parse(&with_scheme).ok()?; + let host = url.host_str()?.to_string(); + let path_segments: Vec = url + .path_segments() + .map(|segs| segs.filter(|s| !s.is_empty()).map(str::to_string).collect()) + .unwrap_or_default(); + + let mut query = HashMap::new(); + for (k, v) in url.query_pairs() { + query.insert(k.into_owned(), v.into_owned()); + } + + Some(Self { + scheme: url.scheme().to_string(), + path_segments, + query, + host, + }) + } + + /// Host normalized for graph entry matching (lowercase, aliases). + pub fn match_host(&self) -> String { + normalize_match_host(&self.host) + } +} + +pub fn normalize_match_host(host: &str) -> String { + let h = host + .strip_prefix("www.") + .unwrap_or(host) + .to_ascii_lowercase(); + match h.as_str() { + "old.reddit.com" | "new.reddit.com" => "reddit.com".to_string(), + "m.youtube.com" => "youtube.com".to_string(), + _ => h, + } +} + +pub fn strip_tracking_query(query: &mut HashMap) { + query.retain(|k, _| { + let lower = k.to_ascii_lowercase(); + !(lower.starts_with("utm_") + || matches!( + lower.as_str(), + "fbclid" | "gclid" | "ref" | "ref_src" | "ref_source" | "mc_cid" | "mc_eid" + )) + }); +} + +#[cfg(test)] +pub(crate) fn test_parts(host: &str, segs: &[&str], query: &[(&str, &str)]) -> UrlParts { + UrlParts { + scheme: "https".to_string(), + host: host.to_string(), + path_segments: segs.iter().map(|s| (*s).to_string()).collect(), + query: query + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_full_url_splits_host_path_query() { + let p = UrlParts::parse("https://www.youtube.com/watch?v=abc&t=4").unwrap(); + assert_eq!(p.host, "www.youtube.com"); + assert_eq!(p.path_segments, vec!["watch"]); + assert_eq!(p.query.get("v").map(String::as_str), Some("abc")); + assert_eq!(p.query.get("t").map(String::as_str), Some("4")); + } + + #[test] + fn parse_r_shortcut_expands_to_reddit() { + let p = UrlParts::parse("r/rust").unwrap(); + assert_eq!(p.match_host(), "reddit.com"); + assert_eq!(p.path_segments, vec!["r", "rust"]); + } + + #[test] + fn parse_slash_r_shortcut() { + let p = UrlParts::parse("/r/aww").unwrap(); + assert_eq!(p.path_segments, vec!["r", "aww"]); + } + + #[test] + fn parse_schemeless_host_path() { + let p = UrlParts::parse("reddit.com/r/rust/comments/aaa/slug").unwrap(); + assert_eq!(p.match_host(), "reddit.com"); + assert_eq!( + p.path_segments, + vec!["r", "rust", "comments", "aaa", "slug"] + ); + } + + #[test] + fn parse_empty_returns_none() { + assert!(UrlParts::parse("").is_none()); + assert!(UrlParts::parse(" ").is_none()); + } + + #[test] + fn normalize_match_host_reddit_aliases() { + assert_eq!(normalize_match_host("old.reddit.com"), "reddit.com"); + assert_eq!(normalize_match_host("NEW.reddit.com"), "reddit.com"); + assert_eq!(normalize_match_host("www.reddit.com"), "reddit.com"); + } + + #[test] + fn normalize_match_host_youtube_aliases() { + assert_eq!(normalize_match_host("m.youtube.com"), "youtube.com"); + assert_eq!(normalize_match_host("www.youtube.com"), "youtube.com"); + } + + #[test] + fn strip_tracking_query_removes_known_params() { + let mut q = HashMap::from([ + ("v".into(), "1".into()), + ("utm_source".into(), "x".into()), + ("fbclid".into(), "y".into()), + ("ref".into(), "z".into()), + ]); + strip_tracking_query(&mut q); + assert_eq!(q.len(), 1); + assert_eq!(q.get("v").map(String::as_str), Some("1")); + } + + #[test] + fn strip_tracking_query_utm_prefix() { + let mut q = HashMap::from([("utm_campaign".into(), "email".into())]); + strip_tracking_query(&mut q); + assert!(q.is_empty()); + } +} diff --git a/server/src/url_rules/registry_tests.rs b/server/src/url_rules/registry_tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..7b76b550f808f590e011469fdae02adf603b43d2 --- /dev/null +++ b/server/src/url_rules/registry_tests.rs @@ -0,0 +1,195 @@ +//! End-to-end tests for the public registry API (`canonicalize_raw`, breadcrumbs, parent). + +use super::registry::{ + canonicalize_raw, looks_like_url, navigable_breadcrumbs, parent_url, resolve_id, +}; + +fn canon(raw: &str) -> String { + canonicalize_raw(raw).unwrap().canonical +} + +#[test] +fn looks_like_url_positive_cases() { + for raw in [ + "https://reddit.com/r/rust", + "r/rust", + "/r/aww", + "reddit.com/r/x", + "www.example.com/path", + "youtu.be/abc", + ] { + assert!(looks_like_url(raw), "{raw}"); + } +} + +#[test] +fn looks_like_url_negative_cases() { + for raw in ["alpha", "beta", "", "hello world", "no-dots"] { + assert!(!looks_like_url(raw), "{raw}"); + } +} + +#[test] +fn resolve_id_matches_canonicalize_raw() { + let raw = "https://youtu.be/xyz"; + assert_eq!( + resolve_id(raw).as_deref(), + Some(canon(raw).as_str()) + ); +} + +#[test] +fn canonicalize_empty_returns_none() { + assert!(canonicalize_raw("").is_none()); +} + +#[test] +fn alias_when_slug_stripped() { + let r = canonicalize_raw( + "https://reddit.com/r/rust/comments/aaa/very_long_title_slug", + ) + .unwrap(); + assert_eq!(r.canonical, "https://reddit.com/r/rust/comments/aaa"); + assert!(r.alias_of.is_some()); +} + +#[test] +fn alias_none_when_already_canonical() { + let raw = "https://reddit.com/r/rust"; + let r = canonicalize_raw(raw).unwrap(); + assert_eq!(r.canonical, raw); + assert!(r.alias_of.is_none()); +} + +#[test] +fn parent_url_subreddit_under_r_hub() { + assert_eq!( + parent_url("https://reddit.com/r/movies").as_deref(), + Some("https://reddit.com/r") + ); +} + +#[test] +fn parent_url_domain_has_none() { + assert_eq!(parent_url("https://reddit.com").as_deref(), None); +} + +#[test] +fn parent_url_generic_site() { + assert_eq!( + parent_url("https://example.com/a/b").as_deref(), + Some("https://example.com/a") + ); +} + +#[test] +fn breadcrumbs_from_canonical_string_roundtrip() { + let id = "https://reddit.com/r/golang/comments/abc123"; + let crumbs = navigable_breadcrumbs(id); + assert_eq!(crumbs.last().map(String::as_str), Some(id)); +} + +// --- Table: Reddit raw URLs → canonical --- + +#[test] +fn reddit_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ("r/rust", "https://reddit.com/r/rust"), + ("/r/aww", "https://reddit.com/r/aww"), + ("https://reddit.com/r/rust", "https://reddit.com/r/rust"), + ( + "https://www.reddit.com/r/programming/new", + "https://reddit.com/r/programming", + ), + ( + "https://old.reddit.com/r/test/comments/xyz/slug/", + "https://reddit.com/r/test/comments/xyz", + ), + ( + "reddit.com/r/Movies/comments/abc/Title_Case_Slug", + "https://reddit.com/r/movies/comments/abc", + ), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Table: YouTube raw URLs → canonical --- + +#[test] +fn youtube_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ( + "https://youtube.com/watch?v=abc", + "https://youtube.com/watch?v=abc", + ), + ( + "https://www.youtube.com/watch?v=abc&t=1&feature=share", + "https://youtube.com/watch?v=abc", + ), + ("https://youtu.be/abc", "https://youtube.com/watch?v=abc"), + ( + "https://youtube.com/shorts/abc", + "https://youtube.com/watch?v=abc", + ), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Table: generic sites --- + +#[test] +fn generic_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ( + "https://news.ycombinator.com/item?id=38472", + "https://news.ycombinator.com/item?id=38472", + ), + ( + "https://www.github.com/rust-lang/rust/issues/1?utm_source=x", + "https://github.com/rust-lang/rust/issues/1", + ), + ("https://example.com", "https://example.com"), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Phantom /comments/ regression (sorter2-specific) --- + +#[test] +fn phantom_comments_not_in_breadcrumbs_for_post() { + let crumbs = navigable_breadcrumbs("https://reddit.com/r/rust/comments/aaa"); + assert!(!crumbs.iter().any(|c| c.ends_with("/comments"))); +} + +#[test] +fn phantom_comments_not_sibling_of_subreddit_in_breadcrumb_chain() { + let crumbs = navigable_breadcrumbs("https://reddit.com/r/rust/comments/aaa"); + let subs: Vec<_> = crumbs + .iter() + .filter(|c| c.contains("/r/rust") && !c.contains("/comments/")) + .collect(); + assert_eq!(subs, vec!["https://reddit.com/r/rust"]); +} + +// --- Distinct items must stay distinct --- + +#[test] +fn different_posts_different_canonical() { + let a = canon("https://reddit.com/r/rust/comments/aaa"); + let b = canon("https://reddit.com/r/rust/comments/bbb"); + assert_ne!(a, b); +} + +#[test] +fn different_subreddits_different_canonical() { + assert_ne!( + canon("https://reddit.com/r/rust"), + canon("https://reddit.com/r/golang") + ); +}