You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [00be3a29] invite system Side A — unified diff (full patch): diff --git a/bb.edn b/bb.edn index 50be8232847e672b3f273a2fb25ddd1d12adb7e2..818f850765370d12d58f239e287764fc3649d78b 100644 --- a/bb.edn +++ b/bb.edn @@ -47,14 +47,16 @@ "RUST_LOG" "info"})})))} test - {:doc "Full test suite: integration + auth + grants" + {:doc "Full test suite: integration + auth + grants + invites" :requires ([test.integration :as integration] [test.auth :as auth] - [test.grants :as grants]) + [test.grants :as grants] + [test.invites :as invites]) :task (do (integration/integration) (auth/auth-test) - (grants/grants-test))} + (grants/grants-test) + (invites/invites-test))} perf {:doc "Performance test: concurrent HTTP requests to detect blocking I/O" diff --git a/cli/src/main.rs b/cli/src/main.rs index e5833b0b93d8e667b94c574ba2b0f8cb758ff3df..8eda9f485bd1f7392f1e34be27176c21e20354eb 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -105,6 +105,23 @@ enum ScopedCmd { #[arg(long)] json: bool, }, + + /// Mint a shareable invite link (24h TTL, in-memory until redeemed). Requires Manage on the room. + InviteLink { + /// Comma-separated: view, post, vote, add_item, manage + #[arg(long = "caps", value_delimiter = ',')] + caps: Vec, + #[arg(long, default_value_t = 1)] + uses: usize, + #[arg(long)] + json: bool, + }, + + /// List principals granted access in this room (requires View or Manage) + Audit { + #[arg(long)] + json: bool, + }, } #[derive(Subcommand, Debug)] @@ -514,17 +531,35 @@ fn print_thread(resp: &ThreadDetailResponse) { .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as i64; - if resp.total > resp.posts.len() { - let end = resp.offset + resp.posts.len(); - eprintln!("# showing {}-{} of {} posts (--offset N --limit N to paginate)", resp.offset, end.saturating_sub(1), resp.total); + if resp.total > resp.items.len() { + let end = resp.offset + resp.items.len(); + eprintln!( + "# showing {}-{} of {} rows (--offset N --limit N to paginate)", + resp.offset, + end.saturating_sub(1), + resp.total + ); } - for (i, post) in resp.posts.iter().enumerate() { - let timeago = slug_types::timeago::timeago_compact(now_ms, post.ts); - let body = &post.body.trim(); - println!("", post.index, timeago); - println!("{}", body); - println!(""); - if i + 1 < resp.posts.len() { + for (i, item) in resp.items.iter().enumerate() { + match item { + ThreadItem::Post { + index, + ts, + body, + .. + } => { + let timeago = slug_types::timeago::timeago_compact(now_ms, *ts); + let body = body.trim(); + println!("", index, timeago); + println!("{}", body); + println!(""); + } + ThreadItem::System { ts, text } => { + let timeago = slug_types::timeago::timeago_compact(now_ms, *ts); + println!("{}", timeago, text.trim()); + } + } + if i + 1 < resp.items.len() { println!(); println!(); } @@ -1036,6 +1071,95 @@ async fn run_scoped(base: &str, room: &str, sub: ScopedCmd) -> Result<()> { } } }, + ScopedCmd::InviteLink { caps, uses, json } => { + let caps: Vec = caps + .into_iter() + .flat_map(|s| { + s.split(',') + .map(|p| p.trim().to_lowercase()) + .filter(|p| !p.is_empty()) + .collect::>() + }) + .collect(); + if caps.is_empty() { + return Err(anyhow!("--caps is required (e.g. --caps view,post,vote)")); + } + let bearer = effective_bearer().ok_or_else(|| { + anyhow!( + "no bearer token: run `slugsocial identity start --rig --model ` \ + then `slugsocial identity poll `, or set SLUG_BEARER_TOKEN / ~/.config/slugsocial/token" + ) + })?; + let batch = send_rpc( + &client, + base, + Some(&bearer), + vec![RpcCommand::RoomMintInvite { + room: room.to_string(), + capabilities: caps, + max_uses: uses, + }], + ) + .await?; + match rpc_line_ok(&batch.results[0])? { + RpcResult::RoomInviteMinted { + invite_url, + expires_at_ms, + max_uses, + } => { + if json { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "invite_url": invite_url, + "expires_at_ms": expires_at_ms, + "max_uses": max_uses, + }))? + ); + } else { + println!("{invite_url}"); + println!("(Expires in 24 hours. Max uses: {max_uses})"); + } + } + _ => return Err(anyhow!("unexpected RPC result")), + } + } + ScopedCmd::Audit { json } => { + let bearer = effective_bearer().ok_or_else(|| { + anyhow!( + "no bearer token: run `slugsocial identity start --rig --model ` \ + then `slugsocial identity poll `, or set SLUG_BEARER_TOKEN / ~/.config/slugsocial/token" + ) + })?; + let batch = send_rpc( + &client, + base, + Some(&bearer), + vec![RpcCommand::RoomAudit { + room: room.to_string(), + }], + ) + .await?; + match rpc_line_ok(&batch.results[0])? { + RpcResult::RoomAudit(resp) => { + if json { + println!("{}", serde_json::to_string_pretty(&resp)?); + } else { + println!("room {}", resp.room); + if resp.grants.is_empty() { + println!("(no grants recorded)"); + } else { + let w_user = resp.grants.iter().map(|g| g.username.len()).max().unwrap_or(0); + for g in &resp.grants { + let caps = g.capabilities.join(", "); + println!("{: return Err(anyhow!("unexpected RPC result")), + } + } ScopedCmd::Check { file, json } => { let mut text = String::new(); match file { diff --git a/server/src/api/auth.rs b/server/src/api/auth.rs index 995ce4a61d29b024c399c656134f541ecfd880cf..b45ba39419c84af8bf2333fc9b7d47e98525c45f 100644 --- a/server/src/api/auth.rs +++ b/server/src/api/auth.rs @@ -12,12 +12,61 @@ use tokio::sync::RwLock; use crate::{ api::helpers::{api_error, now_ms, sha256_hex}, - events::{Event, TokenIssued, UserRegistered}, + events::{Event, GrantAdded, TokenIssued, UserRegistered}, identity::{parse_agent, parse_username}, html::{auth_complete_page, auth_signed_in_fragment, choose_username_error_fragment, choose_username_page}, state::{AppState, PendingSession}, }; +/// Delegate id for browser users who land via `/join/inv_…` (no CLI agent). +const INVITE_BROWSER_AGENT: &str = "00000000-0000-0000-0000-000000000000:invite:web/join"; + +async fn apply_invite_redemption(state: &AppState, invite_token: &str, grantee_username: &str) -> Result<(), String> { + let now = now_ms(); + let ga = { + let mut invites = state.invites.write().await; + let Some(inv) = invites.get_mut(invite_token) else { + return Err("invite not found".into()); + }; + if now > inv.expires_at_ms { + invites.remove(invite_token); + return Err("invite expired".into()); + } + if inv.current_uses >= inv.max_uses { + return Err("invite exhausted".into()); + } + inv.current_uses += 1; + Event::GrantAdded(GrantAdded { + ts: now, + room_id: inv.room_id.clone(), + username: grantee_username.to_string(), + capabilities: inv.capabilities.clone(), + granted_by: inv.inviter.clone(), + }) + }; + + match state.event_log.append(&ga).await { + Ok(()) => { + let mut reduced = state.reduced.write().await; + reduced.apply_event(ga); + let mut invites = state.invites.write().await; + if let Some(inv) = invites.get(invite_token) { + if inv.current_uses >= inv.max_uses { + invites.remove(invite_token); + } + } + Ok(()) + } + Err(e) => { + let mut invites = state.invites.write().await; + if let Some(inv) = invites.get_mut(invite_token) { + inv.current_uses = inv.current_uses.saturating_sub(1); + } + Err(format!("{e}")) + } + } +} + fn pending_sessions(state: &AppState) -> Arc>> { state.pending_sessions.clone() } @@ -115,6 +164,42 @@ pub struct AuthLoginQuery { pub session: String, } +pub async fn get_join_invite(Path(token): Path, State(state): State) -> impl IntoResponse { + let token = token.trim().to_string(); + if token.is_empty() { + return api_error(StatusCode::NOT_FOUND, "invite invalid or expired", None).into_response(); + } + let now = now_ms(); + let valid = { + let invites = state.invites.read().await; + match invites.get(&token) { + None => false, + Some(inv) => now <= inv.expires_at_ms && inv.current_uses < inv.max_uses, + } + }; + if !valid { + return api_error(StatusCode::NOT_FOUND, "invite invalid or expired", None).into_response(); + } + + let session = format!("p_{}", uuid::Uuid::new_v4().simple()); + let s = PendingSession { + agent: INVITE_BROWSER_AGENT.to_string(), + created_ts: now_ms(), + provider: None, + provider_id: None, + redeem_invite: Some(token), + complete: None, + }; + state.pending_sessions.write().await.insert(session.clone(), s); + + let public_url = std::env::var("SLUG_PUBLIC_URL").unwrap_or_else(|_| "http://127.0.0.1:8080".to_string()); + Redirect::temporary(&format!( + "{public_url}/auth/login?session={}", + urlencoding::encode(&session) + )) + .into_response() +} + pub async fn get_auth_login(Query(q): Query, State(state): State) -> impl IntoResponse { // Redirect to Google auth endpoint. let sessions = pending_sessions(&state); @@ -205,6 +290,7 @@ pub async fn get_auth_callback(Query(q): Query, State(state): s.provider = Some("google".to_string()); s.provider_id = Some(sub.clone()); if let Some(username) = existing { + let invite_tok = s.redeem_invite.clone(); let (bearer, token_event) = issue_token_for_user(&username); // append token event let ev = Event::TokenIssued(token_event); @@ -215,6 +301,11 @@ pub async fn get_auth_callback(Query(q): Query, State(state): let mut reduced = reduced_arc.write().await; reduced.apply_event(ev); } + if let Some(tok) = invite_tok { + if let Err(e) = apply_invite_redemption(&state, &tok, &username).await { + tracing::warn!(error = %e, "invite redemption skipped after oauth"); + } + } s.complete = Some((username, bearer)); return Redirect::temporary(&format!("{public_url}/auth/complete")).into_response(); } @@ -310,6 +401,16 @@ pub async fn post_choose_username( reduced.apply_event(ti_ev.clone()); } + // Redeem invite (if any) before marking the session complete. + if let Some(tok) = { + let sessions_read = sessions.read().await; + sessions_read.get(&form.session).and_then(|s| s.redeem_invite.clone()) + } { + if let Err(e) = apply_invite_redemption(&state, &tok, &canon_user).await { + tracing::warn!(error = %e, "invite redemption skipped after registration"); + } + } + // Mark complete for polling. { let mut sessions_write = sessions.write().await; @@ -339,6 +440,7 @@ pub async fn post_pending_session( created_ts: now_ms(), provider: None, provider_id: None, + redeem_invite: None, complete: None, }; let sessions = pending_sessions(&state); diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index a1ea432932cdca0e42dc7377f0a25075fd5644c0..cb031aecebad1107dfa2898a98fb6084b28ba6e9 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -4,6 +4,7 @@ mod rpc; mod validate; pub use auth::{ + get_join_invite, get_pending_session, get_whoami, post_pending_session, diff --git a/server/src/api/rpc.rs b/server/src/api/rpc.rs index 11a76ed7a02aae588edadb7a87a43660f5669d9e..f6bbc3df71909a2da7403cd46fe4ea6ca130c692 100644 --- a/server/src/api/rpc.rs +++ b/server/src/api/rpc.rs @@ -13,12 +13,14 @@ use slug_types::*; use crate::{ canonical_path::{canonicalize_item, canonicalize_tag}, dsl, - events::{AgentBound, Event, GrantAdded, Ingest, RoomCreated, ThreadCapability, ThreadVisibility}, + events::{ + AgentBound, Event, GrantAdded, Ingest, RoomCreated, ThreadCapability, ThreadVisibility, + }, identity::{parse_agent, parse_username}, path_types::CanonicalItemUrl, ranking::{connected_components_from_voted_pairs, ranked_items_subset}, reducer::{scope_from_room_wire, ReducerState, ScopeId}, - state::AppState, + state::{AppState, InviteState}, }; use super::auth::verify_bearer_principal; @@ -142,6 +144,27 @@ fn gen_short_id() -> String { (0..7).map(|_| ALPHABET[rng.gen_range(0..ALPHABET.len())] as char).collect() } +fn gen_invite_token() -> String { + use rand::Rng; + const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + let mut rng = rand::thread_rng(); + let tail: String = (0..16).map(|_| ALPHABET[rng.gen_range(0..ALPHABET.len())] as char).collect(); + format!("inv_{tail}") +} + +const INVITE_TTL_MS: i64 = 86_400_000; + +fn capability_wire(c: ThreadCapability) -> String { + match c { + ThreadCapability::View => "view", + ThreadCapability::Post => "post", + ThreadCapability::Vote => "vote", + ThreadCapability::AddItem => "add_item", + ThreadCapability::Manage => "manage", + } + .to_string() +} + fn build_rank_response_for_content( content: &crate::reducer::ContentState, parent: Option<&str>, @@ -512,7 +535,7 @@ fn rpc_forum_thread_detail( None => Err(("post not found".into(), None)), Some((idx, ing)) => Ok(ThreadDetailResponse { thread: format!("#{}", tag), - posts: vec![PostRow { + items: vec![ThreadItem::Post { id: ing.id.clone(), index: idx, ts: ing.ts, @@ -543,7 +566,7 @@ fn rpc_forum_thread_detail( let total = filtered.len(); const MAX_BODY: usize = 2000; - let posts: Vec = filtered + let items: Vec = filtered .into_iter() .skip(offset) .take(limit) @@ -553,7 +576,7 @@ fn rpc_forum_thread_detail( } else { (ing.raw.clone(), false) }; - PostRow { + ThreadItem::Post { id: ing.id.clone(), index: idx, ts: ing.ts, @@ -566,7 +589,7 @@ fn rpc_forum_thread_detail( Ok(ThreadDetailResponse { thread: format!("#{}", tag), - posts, + items, total, offset, }) @@ -1007,7 +1030,7 @@ pub async fn handle_rpc_batch( RpcCommand::RoomGrant { room, username, - capability, + capabilities, } => { let principal = { let reduced = state.reduced.read().await; @@ -1022,6 +1045,8 @@ pub async fn handle_rpc_batch( }; if !can_manage { line_err("requires Manage capability", None) + } else if capabilities.is_empty() { + line_err("capabilities must not be empty", None) } else { match parse_username(&username) { Err(msg) => line_err("invalid username", Some(msg)), @@ -1033,14 +1058,18 @@ pub async fn handle_rpc_batch( if !user_exists { line_err(format!("user @{target} not found"), None) } else { - match parse_capability(&capability) { + let caps: Result, String> = capabilities + .iter() + .map(|c| parse_capability(c.trim())) + .collect(); + match caps { Err(msg) => line_err(msg, None), - Ok(cap) => { + Ok(caps) => { let ga_ev = Event::GrantAdded(GrantAdded { ts: now_ms(), room_id: room, username: target, - capabilities: vec![cap], + capabilities: caps, granted_by: principal, }); if let Err(e) = state.event_log.append(&ga_ev).await { @@ -1059,6 +1088,114 @@ pub async fn handle_rpc_batch( } } } + RpcCommand::RoomMintInvite { + room, + capabilities, + max_uses, + } => { + let principal = { + let reduced = state.reduced.read().await; + verify_bearer_principal(&headers, &*reduced) + }; + match principal { + Err((_, m)) => line_err(m, None), + Ok(principal) => { + let can_manage = { + let reduced = state.reduced.read().await; + reduced.user_has_cap(&room, &principal, ThreadCapability::Manage) + }; + if !can_manage { + line_err("requires Manage capability", None) + } else if capabilities.is_empty() { + line_err("capabilities must not be empty", None) + } else { + match capabilities + .iter() + .map(|c| parse_capability(c.trim())) + .collect::, String>>() + { + Err(msg) => line_err(msg, None), + Ok(caps) => { + let max_uses = max_uses.max(1).min(100_000); + let now = now_ms(); + let expires_at_ms = now + INVITE_TTL_MS; + let token = loop { + let t = gen_invite_token(); + let taken = { + let invites = state.invites.read().await; + invites.contains_key(&t) + }; + if !taken { + break t; + } + }; + let inv = InviteState { + room_id: room.clone(), + capabilities: caps, + expires_at_ms, + max_uses, + current_uses: 0, + inviter: principal, + }; + state.invites.write().await.insert(token.clone(), inv); + let public_url = std::env::var("SLUG_PUBLIC_URL") + .unwrap_or_else(|_| "http://127.0.0.1:8080".to_string()); + let invite_url = format!("{public_url}/join/{token}"); + line_ok(RpcResult::RoomInviteMinted { + invite_url, + expires_at_ms: Some(expires_at_ms), + max_uses, + }) + } + } + } + } + } + } + RpcCommand::RoomAudit { room } => { + let principal = { + let reduced = state.reduced.read().await; + verify_bearer_principal(&headers, &*reduced) + }; + match principal { + Err((_, m)) => line_err(m, None), + Ok(principal) => { + let reduced = state.reduced.read().await; + if !reduced.rooms.contains_key(&room) { + line_err("unknown room", None) + } else { + let can_audit = reduced.user_has_cap(&room, &principal, ThreadCapability::View) + || reduced.user_has_cap(&room, &principal, ThreadCapability::Manage); + if !can_audit { + line_err("requires View or Manage capability", None) + } else { + let grants: Vec = reduced + .grants + .get(&room) + .map(|m| { + let mut v: Vec = m + .iter() + .map(|(username, caps)| { + let mut c: Vec = + caps.iter().copied().map(capability_wire).collect(); + c.sort(); + RoomAuditEntry { + username: username.clone(), + capabilities: c, + } + }) + .collect(); + v.sort_by(|a, b| a.username.cmp(&b.username)); + v + }) + .unwrap_or_default(); + line_ok(RpcResult::RoomAudit(RoomAuditResponse { room, grants })) + } + } + } + } + } + RpcCommand::RoomRevoke { .. } => line_err("RoomRevoke is not implemented yet", None), RpcCommand::GetGlobalRank { room, limit, diff --git a/server/src/events.rs b/server/src/events.rs index 125005d89ade7b22c9b6d997f0ed81781df5e7ec..9e60f2c218f5c871a93e855e415dd984e3a017c5 100644 --- a/server/src/events.rs +++ b/server/src/events.rs @@ -26,6 +26,8 @@ pub enum Event { RoomCreated(RoomCreated), GrantAdded(GrantAdded), GrantRevoked(GrantRevoked), + InviteMinted(InviteMinted), + InviteRedeemed(InviteRedeemed), /// Ingest of a DSL+prose body. Identity and routing live in event metadata. Ingest(Ingest), } @@ -82,6 +84,25 @@ pub struct GrantRevoked { pub revoked_by: String, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct InviteMinted { + pub ts: i64, + pub token: String, + pub room_id: String, + pub capabilities: Vec, + pub inviter: String, + pub max_uses: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_ts_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct InviteRedeemed { + pub ts: i64, + pub token: String, + pub username: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Ingest { /// Unix timestamp in milliseconds. diff --git a/server/src/lib.rs b/server/src/lib.rs index d00b56d909e1fabd1d122d054dd02785e13e0bd8..bf2f73d4d0f26a357961e88c52c3b4f72626af81 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -27,6 +27,7 @@ pub fn create_app(state: AppState) -> Router { Router::new() .route("/healthz", axum::routing::get(|| async { "ok" })) .route("/static/:filename", axum::routing::get(crate::html::serve_theme_css)) + .route("/join/:token", axum::routing::get(api::get_join_invite)) .route("/auth/login", axum::routing::get(api::get_auth_login)) .route("/auth/callback", axum::routing::get(api::get_auth_callback)) .route("/auth/complete", axum::routing::get(api::get_auth_complete)) diff --git a/server/src/reducer.rs b/server/src/reducer.rs index 118cb19f4df4a083df2878ca7a1424f3d63de851..5190d0a7ee3d545a786a68fef483322aafdce7f9 100644 --- a/server/src/reducer.rs +++ b/server/src/reducer.rs @@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use serde::{Deserialize, Serialize}; use crate::canonical_path::canonicalize_tag; -use crate::events::{Event, Ingest, ThreadCapability}; +use crate::events::{Event, Ingest, ThreadCapability, ThreadVisibility}; use crate::path_types::CanonicalItemUrl; #[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] @@ -156,6 +156,41 @@ pub struct RoomState { pub visibility: crate::events::ThreadVisibility, } +/// Durable invite link state (from [`crate::events::InviteMinted`] / [`crate::events::InviteRedeemed`]). +#[derive(Debug, Clone)] +pub struct ActiveInviteState { + pub room_id: String, + pub capabilities: HashSet, + pub inviter: String, + pub uses_remaining: u32, + pub expires_ts_ms: Option, +} + +#[derive(Clone, Debug)] +pub enum RoomTimelineKind { + RoomCreated { + owner: String, + slug: String, + visibility: ThreadVisibility, + }, + GrantAdded { + username: String, + granted_by: String, + capabilities: Vec, + }, + GrantRevoked { + username: String, + revoked_by: String, + capabilities: Vec, + }, +} + +#[derive(Clone, Debug)] +pub struct RoomTimelineEntry { + pub ts: i64, + pub kind: RoomTimelineKind, +} + #[derive(Debug, Clone)] pub struct ForumThreadState { pub last_activity_ts: i64, @@ -210,6 +245,10 @@ pub struct ReducerState { pub ingests_ordered: Vec, /// room_id → username → capabilities pub grants: HashMap>>, + /// room_id → chronological room admin lines (for thread UI). + pub room_timeline: HashMap>, + /// Invite token → active invite (absent when fully consumed or never minted). + pub invites: HashMap, } impl ReducerState { @@ -225,6 +264,20 @@ impl ReducerState { .unwrap_or(false) } + /// Invite link is present, not expired, and has uses left. + pub fn invite_token_active(&self, token: &str, now_ms: i64) -> Option<&ActiveInviteState> { + let inv = self.invites.get(token)?; + if inv.uses_remaining == 0 { + return None; + } + if let Some(exp) = inv.expires_ts_ms { + if now_ms > exp { + return None; + } + } + Some(inv) + } + pub fn content_for_scope_mut(&mut self, scope: ScopeId) -> &mut ContentState { self.content.entry(scope).or_default() } @@ -356,6 +409,17 @@ impl ReducerState { visibility: rc.visibility, }, ); + self.room_timeline + .entry(rc.room_id.clone()) + .or_default() + .push(RoomTimelineEntry { + ts: rc.ts, + kind: RoomTimelineKind::RoomCreated { + owner: rc.owner.clone(), + slug: rc.slug.clone(), + visibility: rc.visibility, + }, + }); } Event::Ingest(mut ing) => { ing.thread_tag = canonicalize_tag(&ing.thread_tag); @@ -525,18 +589,31 @@ impl ReducerState { nav!(self.actor_last_post_ts, keypath(ing.principal.clone()), setval(ing.ts)); } Event::GrantAdded(ga) => { + let room_id = ga.room_id.clone(); let caps = self.grants .entry(ga.room_id) .or_default() - .entry(ga.username) + .entry(ga.username.clone()) .or_default(); - for cap in ga.capabilities { + for cap in ga.capabilities.iter().copied() { caps.insert(cap); } + self.room_timeline + .entry(room_id) + .or_default() + .push(RoomTimelineEntry { + ts: ga.ts, + kind: RoomTimelineKind::GrantAdded { + username: ga.username.clone(), + granted_by: ga.granted_by.clone(), + capabilities: ga.capabilities.clone(), + }, + }); } Event::GrantRevoked(gr) => { + let room_id = gr.room_id.clone(); if let Some(room_grants) = self.grants.get_mut(&gr.room_id) { - let username = gr.username; + let username = gr.username.clone(); if let Some(caps) = room_grants.get_mut(&username) { for cap in &gr.capabilities { caps.remove(cap); @@ -549,6 +626,37 @@ impl ReducerState { self.grants.remove(&gr.room_id); } } + self.room_timeline + .entry(room_id) + .or_default() + .push(RoomTimelineEntry { + ts: gr.ts, + kind: RoomTimelineKind::GrantRevoked { + username: gr.username.clone(), + revoked_by: gr.revoked_by.clone(), + capabilities: gr.capabilities.clone(), + }, + }); + } + Event::InviteMinted(im) => { + self.invites.insert( + im.token.clone(), + ActiveInviteState { + room_id: im.room_id.clone(), + capabilities: im.capabilities.iter().copied().collect(), + inviter: im.inviter.clone(), + uses_remaining: im.max_uses, + expires_ts_ms: im.expires_ts_ms, + }, + ); + } + Event::InviteRedeemed(ir) => { + if let Some(inv) = self.invites.get_mut(&ir.token) { + inv.uses_remaining = inv.uses_remaining.saturating_sub(1); + if inv.uses_remaining == 0 { + self.invites.remove(&ir.token); + } + } } } } @@ -570,6 +678,8 @@ impl Default for ReducerState { actor_last_post_ts: HashMap::new(), ingests_ordered: Vec::new(), grants: HashMap::new(), + room_timeline: HashMap::new(), + invites: HashMap::new(), } } } diff --git a/server/src/state.rs b/server/src/state.rs index b1ff2330903780cbe0cdf35964cb166eb2423d7d..628fd5921b34ba53ac0ca6b6fd33957ca09129b9 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -1,8 +1,20 @@ +use std::collections::HashMap; use std::sync::Arc; use tokio::sync::{broadcast, RwLock}; -use crate::{event_log::EventLog, reducer::ReducerState}; +use crate::{event_log::EventLog, events::ThreadCapability, reducer::ReducerState}; + +/// Ephemeral invite link (24h TTL, in-memory only; not written to the event log). +#[derive(Debug, Clone)] +pub struct InviteState { + pub room_id: String, + pub capabilities: Vec, + pub expires_at_ms: i64, + pub max_uses: usize, + pub current_uses: usize, + pub inviter: String, +} #[derive(Debug, Clone)] pub struct PendingSession { @@ -10,6 +22,8 @@ pub struct PendingSession { pub created_ts: i64, pub provider: Option, pub provider_id: Option, + /// When set, successful OAuth completion redeems this invite token and appends [`crate::events::GrantAdded`]. + pub redeem_invite: Option, pub complete: Option<(String /*username*/, String /*bearer*/ )>, } @@ -42,7 +56,9 @@ pub struct AppState { pub cfg: Arc, pub event_log: Arc, pub reduced: Arc>, - pub pending_sessions: Arc>>, + pub pending_sessions: Arc>>, + /// Ephemeral invite tokens (`inv_…`) until expiry or exhaustion. + pub invites: Arc>>, /// Broadcast channel for SSE live-streaming. Capacity = 64 events. pub stream_tx: broadcast::Sender, /// Broadcast channel for web SSE HTML fragments (poem pattern). Capacity = 64. @@ -58,7 +74,8 @@ impl AppState { cfg: Arc::new(cfg), event_log: Arc::new(event_log), reduced: Arc::new(RwLock::new(ReducerState::default())), - pending_sessions: Arc::new(RwLock::new(std::collections::HashMap::new())), + pending_sessions: Arc::new(RwLock::new(HashMap::new())), + invites: Arc::new(RwLock::new(HashMap::new())), stream_tx, html_tx, } diff --git a/server/src/timeline.rs b/server/src/timeline.rs new file mode 100644 index 0000000000000000000000000000000000000000..251158943ab36d3015268e35f6bdd01c3f42ab3b --- /dev/null +++ b/server/src/timeline.rs @@ -0,0 +1,153 @@ +//! Room admin lines merged into forum thread views. + +use crate::{ + canonical_path::canonicalize_tag, + reducer::{ReducerState, RoomTimelineEntry, RoomTimelineKind}, +}; + +fn cap_label(c: crate::events::ThreadCapability) -> &'static str { + use crate::events::ThreadCapability::*; + match c { + View => "view", + Post => "post", + Vote => "vote", + AddItem => "add_item", + Manage => "manage", + } +} + +fn caps_list(caps: &[crate::events::ThreadCapability]) -> String { + let mut v: Vec<_> = caps.iter().map(|c| cap_label(*c)).collect(); + v.sort(); + v.join(", ") +} + +/// Human-readable system line for the thread feed. +pub fn format_room_timeline_entry(e: &RoomTimelineEntry) -> String { + match &e.kind { + RoomTimelineKind::RoomCreated { + owner, + slug, + visibility, + } => { + let vis = match visibility { + crate::events::ThreadVisibility::Public => "public", + crate::events::ThreadVisibility::Private => "private", + }; + format!("@{owner} created room #{slug} ({vis})") + } + RoomTimelineKind::GrantAdded { + username, + granted_by, + capabilities, + } => { + format!( + "@{granted_by} granted @{} {}", + username, + caps_list(capabilities) + ) + } + RoomTimelineKind::GrantRevoked { + username, + revoked_by, + capabilities, + } => { + format!( + "@{revoked_by} revoked @{} {}", + username, + caps_list(capabilities) + ) + } + } +} + +#[derive(Clone, Debug)] +pub enum MergedThreadRow { + System { ts: i64, text: String }, + Post { + index: usize, + id: String, + ts: i64, + principal: String, + raw: String, + }, +} + +/// Merge room admin lines with thread ingests for one room + tag. Oldest first. +/// `actor_prefix` filters posts only (system lines always included). +pub fn merge_thread_rows( + reduced: &ReducerState, + room_wire: &str, + thread_tag: &str, + since: Option, + before: Option, + actor_prefix: &str, +) -> Vec { + let scope = crate::reducer::scope_from_room_wire(room_wire); + let tag = canonicalize_tag(thread_tag); + let key = (scope.clone(), tag.clone()); + + let mut rows: Vec = Vec::new(); + + if let Some(entries) = reduced.room_timeline.get(room_wire.trim()) { + for e in entries { + if since.map_or(true, |s| e.ts >= s) && before.map_or(true, |b| e.ts < b) { + rows.push(MergedThreadRow::System { + ts: e.ts, + text: format_room_timeline_entry(e), + }); + } + } + } + + let all_ids: Vec = reduced + .ingests_by_scope_thread + .get(&key) + .map(|q| q.iter().rev().cloned().collect()) + .unwrap_or_default(); + + for (idx, id) in all_ids.into_iter().enumerate() { + let Some(ing) = reduced.ingests_by_id.get(&id) else { + continue; + }; + if since.map_or(true, |s| ing.ts >= s) && before.map_or(true, |b| ing.ts < b) { + if !actor_prefix.is_empty() + && !ing + .principal + .to_lowercase() + .starts_with(actor_prefix) + { + continue; + } + rows.push(MergedThreadRow::Post { + index: idx, + id: ing.id.clone(), + ts: ing.ts, + principal: ing.principal.clone(), + raw: ing.raw.clone(), + }); + } + } + + rows.sort_by(|a, b| { + let ta = match a { + MergedThreadRow::System { ts, .. } | MergedThreadRow::Post { ts, .. } => *ts, + }; + let tb = match b { + MergedThreadRow::System { ts, .. } | MergedThreadRow::Post { ts, .. } => *ts, + }; + ta.cmp(&tb) + }); + rows +} + +/// Public forum thread (`room_wire == "public"`): same merge (timeline usually empty). +pub fn merge_public_thread_rows( + reduced: &ReducerState, + thread_tag: &str, + since: Option, + before: Option, + actor_prefix: &str, +) -> Vec { + merge_thread_rows(reduced, "public", thread_tag, since, before, actor_prefix) +} diff --git a/test/grants.bb b/test/grants.bb index 10ef0f08ce1c3945e5f1634314e6aebcf6810cd2..4ded5ce92576cdea653cce0ba8a751be4374731a 100644 --- a/test/grants.bb +++ b/test/grants.bb @@ -105,7 +105,7 @@ ;; Alice grants bob View only. (println "\nalice grants bob View only…") (assert! (rpc-line-ok? (:parsed (rpc-batch! base-url alice-token - [{"RoomGrant" {"room" room-id "username" "bob" "capability" "view"}}]))) + [{"RoomGrant" {"room" room-id "username" "bob" "capabilities" ["view"]}}]))) "grant View RPC ok") (println "\nbob (View only) tries to post prose…") @@ -117,7 +117,7 @@ ;; Alice grants bob Post. (println "\nalice grants bob Post…") (assert! (rpc-line-ok? (:parsed (rpc-batch! base-url alice-token - [{"RoomGrant" {"room" room-id "username" "bob" "capability" "post"}}]))) + [{"RoomGrant" {"room" room-id "username" "bob" "capabilities" ["post"]}}]))) "grant Post RPC ok") (println "\nbob (View + Post) posts prose…") @@ -143,7 +143,7 @@ ;; Alice grants bob Vote. (println "\nalice grants bob Vote…") (assert! (rpc-line-ok? (:parsed (rpc-batch! base-url alice-token - [{"RoomGrant" {"room" room-id "username" "bob" "capability" "vote"}}]))) + [{"RoomGrant" {"room" room-id "username" "bob" "capabilities" ["vote"]}}]))) "grant Vote RPC ok") (println "\nbob (View + Post + Vote) votes…") diff --git a/test/invites.bb b/test/invites.bb new file mode 100644 index 0000000000000000000000000000000000000000..f72b1e630a6856174cdef11eb977e2f81b16e551 --- /dev/null +++ b/test/invites.bb @@ -0,0 +1,150 @@ +(ns test.invites + "Ephemeral invite links: mint via RPC, GET /join → pending session + OAuth, redemption → GrantAdded, + RoomAudit, post succeeds, second GET /join returns 404 when max_uses exhausted." + (:require [babashka.fs :as fs] + [cheshire.core :as json] + [clojure.string :as str] + [test.common :as common] + [test.oauth :as oauth])) + +(def ^:private counts (atom {:pass 0 :fail 0})) + +(defn- assert! [pred msg] + (common/test-assert! counts pred msg)) + +(defn- bearer [token] {"Authorization" (str "Bearer " token)}) + +(defn- rpc-batch! [base-url token cmds] + (let [resp (oauth/http-post-json (str base-url "/api/v0/rpc") cmds :headers (bearer token))] + {:status (:status resp) + :parsed (json/parse-string (:body resp) false)})) + +(defn- rpc-line-ok? [parsed] + (true? (get-in parsed ["results" 0 "ok"]))) + +(defn- session-from-login-location [loc] + (when loc + (let [qpart (if (str/includes? loc "?") + (-> loc (str/split #"\?" 2) second) + "") + qpart (if (str/includes? qpart "#") + (-> qpart (str/split #"#" 2) first) + qpart) + m (oauth/parse-query qpart)] + (some-> (get m :session) str)))) + +(defn- invite-token-from-url [invite-url] + (second (re-find #"/join/(inv_[^/?#]+)" (str invite-url)))) + +(defn- register-user! [base-url session-agent username] + (oauth/complete-registration! base-url + :agent session-agent + :username username + :assert! (fn [pred msg] (assert! pred msg)))) + +(defn- ingest! [base-url token room thread delegate text] + (rpc-batch! base-url token + [{"Post" {"room" room + "thread_tag" thread + "delegate" delegate + "text" text + "return_rank_diff" false}}])) + +(defn invites-test [& _args] + (println "\n━━━ ephemeral invite + audit integration check ━━━\n") + (reset! counts {:pass 0 :fail 0}) + + (println "building server binary…") + (common/letlocals + (bind build (common/run-cargo-build-release! ["slugsocial-server"])) + (assert! (zero? (:exit build)) "cargo build succeeds") + (bind server-bin "target/release/slugsocial-server") + + (bind tmp-dir (str (fs/create-temp-dir {:prefix "slug-invites-"}))) + (bind slug-port (common/pick-port)) + (bind google-port (common/pick-port)) + (bind base-url (str "http://127.0.0.1:" slug-port)) + (bind google-url (str "http://127.0.0.1:" google-port)) + + (bind !server (atom nil)) + (bind !google (atom nil)) + + (bind server-env (common/slug-server-env tmp-dir base-url google-url slug-port)) + (try + (println (str "starting mock google on :" google-port)) + (reset! !google (oauth/start-mock-google google-port + :google-users ["google-user-alice" "google-user-bob"])) + + (println (str "starting server on :" slug-port)) + (reset! !server (common/start-server server-bin server-env)) + (assert! (common/wait-for-server base-url 10000) "server responds to /healthz") + + (println "\nregistering alice…") + (let [alice-token (register-user! base-url + "00000000-0000-0000-0000-000000000001:test:local/dev" + "alice") + + _ (println "\nalice creates private room…") + create (rpc-batch! base-url alice-token + [{"RoomCreate" {"slug" "invite-demo" "visibility" "private"}}]) + _ (assert! (= 200 (:status create)) "room create HTTP 200") + _ (assert! (rpc-line-ok? (:parsed create)) "room create RPC ok") + room-id (get-in (:parsed create) ["results" 0 "result" "RoomCreated" "room_id"]) + _ (assert! (some? room-id) "room_id present") + + _ (println "\nalice mints invite (view,post,vote uses=1)…") + mint (rpc-batch! base-url alice-token + [{"RoomMintInvite" {"room" room-id + "capabilities" ["view" "post" "vote"] + "max_uses" 1}}]) + _ (assert! (= 200 (:status mint)) "mint HTTP 200") + _ (assert! (rpc-line-ok? (:parsed mint)) "mint RPC ok") + invite-url (get-in (:parsed mint) ["results" 0 "result" "RoomInviteMinted" "invite_url"]) + inv-tok (invite-token-from-url invite-url) + _ (assert! (some? inv-tok) "invite token parsed from URL") + + _ (println "\nGET /join/:token (expect redirect + session)…") + join-resp (oauth/http-get-no-redirect (str base-url "/join/" inv-tok)) + _ (assert! (contains? #{302 307} (:status join-resp)) + (str "join returns redirect (got status " (:status join-resp) ")")) + sess (session-from-login-location (:location join-resp)) + _ (assert! (and (some? sess) (str/starts-with? sess "p_")) "Location carries session=p_…") + + _ (println "\nbob completes OAuth via invite session…") + bob-token (oauth/complete-pending-session! base-url sess "bob" + :assert! (fn [pred msg] (assert! pred msg))) + + _ (println "\nalice runs RoomAudit…") + audit (rpc-batch! base-url alice-token [{"RoomAudit" {"room" room-id}}]) + _ (assert! (rpc-line-ok? (:parsed audit)) "audit RPC ok") + grants (get-in (:parsed audit) ["results" 0 "result" "RoomAudit" "grants"]) + bob-entry (first (filter #(= "bob" (get % "username")) grants)) + _ (assert! (some? bob-entry) "audit lists bob") + bob-caps (set (get bob-entry "capabilities")) + _ (assert! (= bob-caps #{"view" "post" "vote"}) "bob has view, post, vote") + + _ (println "\nbob posts prose to private room…") + _ (assert! (rpc-line-ok? (:parsed (ingest! base-url bob-token room-id "main" + "00000000-0000-0000-0000-000000000002:test:local/dev" + "Hello via invite link."))) + "bob post succeeds") + + _ (println "\nsecond GET /join (invite exhausted → 404)…") + join2 (oauth/http-get-no-redirect (str base-url "/join/" inv-tok)) + _ (assert! (= 404 (:status join2)) "exhausted invite returns 404")] + + (println "\ninvite lifecycle OK.")) + + (finally + (when-some [s @!server] (common/kill-server s)) + (when-some [g @!google] ((:stop-fn g))) + (fs/delete-tree tmp-dir))) + + (bind {pass :pass fail :fail} @counts) + (if (zero? fail) + (println (str "\n" common/ansi-green "━━━ " pass " invite checks passed ━━━" common/ansi-reset "\n")) + (do (println (str "\n" common/ansi-red "━━━ " fail " invite checks FAILED ━━━" common/ansi-reset "\n")) + (System/exit 1))))) + +(when (= *file* (System/getProperty "babashka.file")) + (invites-test)) diff --git a/test/oauth.bb b/test/oauth.bb index b69459acd17c23c023170d8dd93fe45bb49c8d88..efb3f3e66346ffde81bf9622d75c3a6ec89f8db2 100644 --- a/test/oauth.bb +++ b/test/oauth.bb @@ -21,6 +21,21 @@ resp (.send (http-client) req (java.net.http.HttpResponse$BodyHandlers/ofString))] {:status (.statusCode resp) :body (.body resp) :headers (.map (.headers resp))}))) +(defn http-get-no-redirect + "GET without following redirects; returns `:location` from the first `Location` header when present." + [url & {:keys [headers]}] + (let [client (-> (java.net.http.HttpClient/newBuilder) + (.followRedirects java.net.http.HttpClient$Redirect/NEVER) + (.connectTimeout connect-timeout) + (.build)) + b (java.net.http.HttpRequest/newBuilder (java.net.URI/create url))] + (doseq [[k v] (or headers {})] + (.header b k v)) + (let [req (-> b (.timeout request-timeout) (.GET) (.build)) + resp (.send client req (java.net.http.HttpResponse$BodyHandlers/ofString)) + loc (first (get (.map (.headers resp)) "location"))] + {:status (.statusCode resp) :body (.body resp) :location loc}))) + (defn http-post-json [url data & {:keys [headers]}] (let [body (json/generate-string data) b (java.net.http.HttpRequest/newBuilder (java.net.URI/create url))] @@ -50,6 +65,35 @@ resp (.send (http-client) req (java.net.http.HttpResponse$BodyHandlers/ofString))] {:status (.statusCode resp) :body (.body resp) :headers (.map (.headers resp))}))) +(defn complete-pending-session! + "Finish OAuth for an existing pending session id (e.g. created by `GET /join/inv_…`). Returns bearer token." + [base-url session-id username & {:keys [assert!]}] + (let [check! (fn [pred msg resp] + (if assert! + (assert! pred msg) + (when-not pred + (throw (ex-info msg {:resp resp})))))] + (let [enc (java.net.URLEncoder/encode session-id "UTF-8") + login-url (str base-url "/auth/login?session=" enc) + login-get (http-get login-url)] + (check! (= 200 (:status login-get)) + (str "oauth redirect chain for session " session-id) + login-get) + (let [choose (http-post-form (str base-url "/auth/choose-username") + {:session session-id :username username})] + (check! (= 200 (:status choose)) + (str "choose-username for " username " returns 200") + choose) + (let [poll (http-get (str base-url "/api/v0/pending-session/" session-id))] + (check! (= 200 (:status poll)) + (str "pending-session poll returns 200") + poll) + (let [poll-json (json/parse-string (:body poll) true)] + (check! (:complete poll-json) + (str "pending session complete for " username) + poll) + (:token poll-json))))))) + (defn parse-query [s] (into {} (for [part (str/split (or s "") #"&") diff --git a/types/src/lib.rs b/types/src/lib.rs index a715fe5639f4a3a7bfedcc706b5fb312b98c9ac5..98c66a00fd27803ef6f75b5ac478ff2eb762d771 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -123,13 +123,32 @@ pub struct PathDetailResponse { #[derive(Debug, Serialize, Deserialize)] pub struct ThreadDetailResponse { pub thread: String, - pub posts: Vec, - /// Total posts in this thread. + /// Chronological page: prose posts and room system lines, oldest first within the window. + pub items: Vec, + /// Total rows (posts + system lines) in this thread after filters. pub total: usize, - /// Chronological offset of the first post in this page. + /// Offset into the merged chronological list. pub offset: usize, } +/// One row in a thread timeline: a normal post or a room system line. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ThreadItem { + Post { + id: String, + index: usize, + ts: i64, + actor: String, + body: String, + truncated: bool, + }, + System { + ts: i64, + text: String, + }, +} + /// One post in a thread. Full body, no snippet. #[derive(Debug, Serialize, Deserialize)] pub struct PostRow { @@ -224,6 +243,23 @@ pub struct FeedPost { // RPC batch API (`POST /api/v0/rpc`) // --------------------------------------------------------------------------- +fn default_invite_max_uses() -> usize { + 1 +} + +/// One principal's capabilities in a private room (from [`RpcCommand::RoomAudit`]). +#[derive(Debug, Serialize, Deserialize)] +pub struct RoomAuditEntry { + pub username: String, + pub capabilities: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct RoomAuditResponse { + pub room: String, + pub grants: Vec, +} + #[derive(Debug, Serialize, Deserialize)] #[serde(transparent)] pub struct RpcBatch(pub Vec); @@ -287,10 +323,27 @@ pub enum RpcCommand { visibility: Option, }, RoomGrant { + room: String, + username: String, + /// Capability names: `view`, `post`, `vote`, `add_item`, `manage`. + capabilities: Vec, + }, + RoomRevoke { room: String, username: String, capability: String, }, + /// Mint a shareable invite link (24h TTL, stored in memory only until redeemed or expiry). + RoomMintInvite { + room: String, + capabilities: Vec, + #[serde(default = "default_invite_max_uses")] + max_uses: usize, + }, + /// List principals granted access in a room (requires View or Manage). + RoomAudit { + room: String, + }, GetGlobalRank { room: String, #[serde(default)] @@ -361,6 +414,13 @@ pub enum RpcResult { RoomCreated { room_id: String, }, + RoomInviteMinted { + invite_url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + expires_at_ms: Option, + max_uses: usize, + }, + RoomAudit(RoomAuditResponse), GrantOk {}, GlobalRank(GlobalRankResponse), Pair(PairResponse), Side B — contributor: tommy-mor Side B — commit message: [15e1037a] url stuff Side B — unified diff (full patch): diff --git a/server/src/url_rules/graph.rs b/server/src/url_rules/graph.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7ac0f9a551a1727cb2f9294778c283b9885b147 --- /dev/null +++ b/server/src/url_rules/graph.rs @@ -0,0 +1,831 @@ +//! Semantic URL graph: DFA traversal on host + path, query in context, generic fallback. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use url::Url; + +use super::graph_builder::GraphBuilder; +use super::parse::{normalize_match_host, strip_tracking_query, UrlParts}; + +#[derive(Debug, Clone, Default)] +pub struct Context { + pub vars: HashMap, + pub query: HashMap, +} + +pub type CanonicalFn = fn(&Context) -> Option; + +#[derive(Clone, Copy)] +pub enum EdgePattern { + Literal(&'static str), + Variable(&'static str), + /// Absorb any trailing segment without leaving this node (e.g. post title slug). + AbsorbAny, + /// Absorb segment when `cond(seg)` (e.g. subreddit listing suffix). + AbsorbIf(fn(&str) -> bool), +} + +pub struct Edge { + pub pattern: EdgePattern, + pub target: &'static str, +} + +pub struct Node { + pub edges: Vec, + pub canonical: CanonicalFn, + pub parent: Option<&'static str>, +} + +impl Node { + pub(crate) fn empty() -> Self { + Self { + edges: Vec::new(), + canonical: |_| None, + parent: None, + } + } +} + +pub struct Graph { + pub nodes: HashMap<&'static str, Node>, +} + +static GRAPH: OnceLock = OnceLock::new(); + +pub fn graph() -> &'static Graph { + GRAPH.get_or_init(build_graph) +} + +impl Graph { + pub fn resolve_canonical(&self, parts: &UrlParts) -> Option { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + + if let Some(node_id) = self.traverse(parts, &mut ctx) { + if let Some(canon) = (self.nodes.get(node_id)?.canonical)(&ctx) { + return Some(canon); + } + } + Some(generic_canonical(parts)) + } + + pub fn breadcrumbs(&self, parts: &UrlParts) -> Vec { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + + if let Some(mut node_id) = self.traverse(parts, &mut ctx) { + let mut paths = Vec::new(); + loop { + let node = match self.nodes.get(node_id) { + Some(n) => n, + None => break, + }; + if let Some(url) = (node.canonical)(&ctx) { + if paths.last() != Some(&url) { + paths.push(url); + } + } + match node.parent { + Some(p) => node_id = p, + None => break, + } + } + paths.reverse(); + if !paths.is_empty() { + return paths; + } + } + generic_breadcrumbs(parts) + } + + fn traverse(&self, parts: &UrlParts, ctx: &mut Context) -> Option<&'static str> { + let host = parts.match_host(); + let mut node_id = match host.as_str() { + "reddit.com" => "reddit_root", + "youtube.com" => "youtube_root", + "youtu.be" => "youtu_be_entry", + _ => return None, + }; + + let segs: Vec<&str> = parts.path_segments.iter().map(String::as_str).collect(); + let mut i = 0; + while i < segs.len() { + let seg = segs[i]; + match self.follow_edge(node_id, seg, ctx) { + Ok(next) => { + node_id = next; + i += 1; + } + Err(()) => { + if self.try_absorb(node_id, seg) { + i += 1; + continue; + } + return None; + } + } + } + Some(node_id) + } + + fn follow_edge( + &self, + node_id: &'static str, + seg: &str, + ctx: &mut Context, + ) -> Result<&'static str, ()> { + let node = self.nodes.get(node_id).ok_or(())?; + for edge in &node.edges { + match edge.pattern { + EdgePattern::Literal(lit) if lit == seg => return Ok(edge.target), + EdgePattern::Variable(name) => { + ctx.vars.insert(name.to_string(), seg.to_string()); + return Ok(edge.target); + } + EdgePattern::AbsorbAny + | EdgePattern::AbsorbIf(_) + | EdgePattern::Literal(_) + | EdgePattern::Variable(_) => {} + } + } + Err(()) + } + + fn try_absorb(&self, node_id: &'static str, seg: &str) -> bool { + let node = match self.nodes.get(node_id) { + Some(n) => n, + None => return false, + }; + for edge in &node.edges { + match edge.pattern { + EdgePattern::AbsorbAny => return true, + EdgePattern::AbsorbIf(cond) if cond(seg) => return true, + EdgePattern::AbsorbIf(_) | EdgePattern::Literal(_) | EdgePattern::Variable(_) => {} + } + } + false + } + + /// Test hook: terminal graph node and captured context after traversal. + #[cfg(test)] + pub fn traverse_terminal(&self, parts: &UrlParts) -> Option<(&'static str, Context)> { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + let node = self.traverse(parts, &mut ctx)?; + Some((node, ctx)) + } +} + +fn is_reddit_listing_suffix(seg: &str) -> bool { + matches!(seg, "hot" | "top" | "new" | "rising" | "controversial") +} + +/// Percent-encode a path or query fragment so `&`, `?`, etc. cannot break URL structure. +fn enc(s: &str) -> String { + urlencoding::encode(s).into_owned() +} + +// --- Canonical formatters --- + +fn canon_reddit_root(_: &Context) -> Option { + Some("https://reddit.com".to_string()) +} + +fn canon_reddit_r_hub(_: &Context) -> Option { + Some("https://reddit.com/r".to_string()) +} + +fn canon_reddit_subreddit(ctx: &Context) -> Option { + let sub = ctx.vars.get("subreddit")?; + Some(format!( + "https://reddit.com/r/{}", + enc(&sub.to_ascii_lowercase()) + )) +} + +fn canon_reddit_post(ctx: &Context) -> Option { + let sub = ctx.vars.get("subreddit")?.to_ascii_lowercase(); + let id = ctx.vars.get("post_id")?; + Some(format!( + "https://reddit.com/r/{}/comments/{}", + enc(&sub), + enc(id) + )) +} + +fn canon_youtube_root(_: &Context) -> Option { + Some("https://youtube.com".to_string()) +} + +fn canon_youtube_watch(ctx: &Context) -> Option { + let v = ctx + .query + .get("v") + .or_else(|| ctx.vars.get("video_id"))?; + Some(format!("https://youtube.com/watch?v={}", enc(v))) +} + +fn canon_youtu_be(ctx: &Context) -> Option { + let v = ctx.vars.get("vid_id")?; + Some(format!("https://youtube.com/watch?v={}", enc(v))) +} + +pub fn build_graph() -> Graph { + GraphBuilder::new() + .node("reddit_root") + .canonical(canon_reddit_root) + .edge(EdgePattern::Literal("r"), "reddit_r_hub") + .node("reddit_r_hub") + .parent("reddit_root") + .canonical(canon_reddit_r_hub) + .edge(EdgePattern::Variable("subreddit"), "reddit_subreddit") + .node("reddit_subreddit") + .parent("reddit_r_hub") + .canonical(canon_reddit_subreddit) + .edge( + EdgePattern::AbsorbIf(is_reddit_listing_suffix), + "reddit_subreddit", + ) + .edge(EdgePattern::Literal("comments"), "reddit_comments_gate") + .node("reddit_comments_gate") + .parent("reddit_subreddit") + .canonical(canon_reddit_subreddit) + .edge(EdgePattern::Variable("post_id"), "reddit_post") + .node("reddit_post") + .parent("reddit_subreddit") + .canonical(canon_reddit_post) + .edge(EdgePattern::AbsorbAny, "reddit_post") + .node("youtube_root") + .canonical(canon_youtube_root) + .edge(EdgePattern::Literal("watch"), "youtube_watch") + .edge(EdgePattern::Literal("shorts"), "youtube_shorts_gate") + .node("youtube_watch") + .parent("youtube_root") + .canonical(canon_youtube_watch) + .node("youtube_shorts_gate") + .parent("youtube_root") + .canonical(canon_youtube_root) + .edge(EdgePattern::Variable("video_id"), "youtube_watch") + .node("youtu_be_entry") + .canonical(canon_youtube_root) + .edge(EdgePattern::Variable("vid_id"), "youtu_be_video") + .node("youtu_be_video") + .parent("youtube_root") + .canonical(canon_youtu_be) + .build() +} + +// --- Generic internet fallback --- + +pub fn generic_canonical(parts: &UrlParts) -> String { + let host = normalize_match_host(&parts.host); + let path_segments: Vec = parts.path_segments.clone(); + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + + let mut url = if path_segments.is_empty() { + Url::parse(&format!("https://{host}")) + .unwrap_or_else(|_| Url::parse("https://invalid").unwrap()) + } else { + let path = format!("/{}", path_segments.join("/")); + Url::parse(&format!("https://{host}{path}")) + .unwrap_or_else(|_| Url::parse("https://invalid").unwrap()) + }; + + if !query.is_empty() { + let mut pairs: Vec<_> = query.iter().collect(); + pairs.sort_by(|a, b| a.0.cmp(b.0)); + url.query_pairs_mut().clear(); + for (k, v) in pairs { + url.query_pairs_mut().append_pair(k, v); + } + } + + let mut s = url.to_string(); + if path_segments.is_empty() { + s = s.trim_end_matches('/').to_string(); + } + s +} + +pub fn generic_breadcrumbs(parts: &UrlParts) -> Vec { + let host = normalize_match_host(&parts.host); + let n = parts.path_segments.len(); + let mut out = Vec::new(); + + let base = generic_canonical(&UrlParts { + scheme: "https".to_string(), + host: host.clone(), + path_segments: vec![], + query: HashMap::new(), + }); + out.push(base); + + for i in 0..n { + let segs: Vec = parts.path_segments[..=i].to_vec(); + let url = generic_canonical(&UrlParts { + scheme: "https".to_string(), + host: host.clone(), + path_segments: segs, + query: HashMap::new(), + }); + if out.last() != Some(&url) { + out.push(url); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::url_rules::parse::test_parts; + + fn g() -> &'static Graph { + graph() + } + + fn canon(parts: &UrlParts) -> String { + g().resolve_canonical(parts).unwrap() + } + + fn crumbs(parts: &UrlParts) -> Vec { + g().breadcrumbs(parts) + } + + fn terminal(parts: &UrlParts) -> Option<&'static str> { + g().traverse_terminal(parts).map(|(n, _)| n) + } + + fn vars(parts: &UrlParts) -> HashMap { + g().traverse_terminal(parts) + .map(|(_, c)| c.vars) + .unwrap_or_default() + } + + #[test] + fn youtu_be_malicious_segment_encoded_not_injected() { + let p = test_parts("youtu.be", &["abc&t=1"], &[]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=abc%26t%3D1"); + assert!(!canon(&p).contains("abc&t=1")); + } + + #[test] + fn youtube_query_v_encoded() { + let p = test_parts("youtube.com", &["watch"], &[("v", "a&b=c")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=a%26b%3Dc"); + } + + #[test] + fn absorb_patterns_live_on_edges_not_in_engine() { + let g = build_graph(); + let sub = g.nodes.get("reddit_subreddit").unwrap(); + assert!(sub + .edges + .iter() + .any(|e| matches!(e.pattern, EdgePattern::AbsorbIf(_)))); + let post = g.nodes.get("reddit_post").unwrap(); + assert!(post + .edges + .iter() + .any(|e| matches!(e.pattern, EdgePattern::AbsorbAny))); + } + + #[test] + fn builder_rejects_missing_parent() { + let result = std::panic::catch_unwind(|| { + GraphBuilder::new() + .node("orphan") + .parent("nonexistent_parent") + .build(); + }); + assert!(result.is_err()); + } + + #[test] + fn generic_canonical_forces_https_and_strips_www() { + let p = test_parts("www.example.com", &["blog", "post"], &[]); + assert_eq!(canon(&p), "https://example.com/blog/post"); + } + + #[test] + fn generic_canonical_sorts_query_keys() { + let p = test_parts("example.com", &["search"], &[("q", "rust"), ("page", "2")]); + assert_eq!(canon(&p), "https://example.com/search?page=2&q=rust"); + } + + #[test] + fn generic_canonical_strips_tracking_from_query() { + let p = test_parts( + "news.ycombinator.com", + &["item"], + &[("id", "1"), ("utm_medium", "social")], + ); + assert_eq!(canon(&p), "https://news.ycombinator.com/item?id=1"); + } + + #[test] + fn generic_breadcrumbs_cumulative_path() { + let p = test_parts("paulgraham.com", &["articles", "lisp.html"], &[]); + assert_eq!( + crumbs(&p), + vec![ + "https://paulgraham.com", + "https://paulgraham.com/articles", + "https://paulgraham.com/articles/lisp.html" + ] + ); + } + + #[test] + fn generic_breadcrumbs_domain_only() { + let p = test_parts("example.com", &[], &[]); + assert_eq!(crumbs(&p), vec!["https://example.com"]); + } + + #[test] + fn unknown_host_uses_generic_not_graph() { + let p = test_parts("hackernews.com", &["item", "123"], &[]); + assert_eq!(terminal(&p), None); + assert_eq!(canon(&p), "https://hackernews.com/item/123"); + } + + #[test] + fn traverse_captures_subreddit_variable() { + let p = test_parts("reddit.com", &["r", "Rust"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(vars(&p).get("subreddit").map(String::as_str), Some("Rust")); + } + + #[test] + fn traverse_captures_post_id() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "abc123"], &[]); + assert_eq!(terminal(&p), Some("reddit_post")); + assert_eq!(vars(&p).get("post_id").map(String::as_str), Some("abc123")); + } + + #[test] + fn traverse_absorbs_listing_suffix_stays_on_subreddit() { + let p = test_parts("reddit.com", &["r", "rust", "hot"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn traverse_absorbs_all_listing_suffixes() { + for suffix in ["hot", "top", "new", "rising", "controversial"] { + let p = test_parts("reddit.com", &["r", "test", suffix], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit"), "suffix {suffix}"); + assert_eq!(canon(&p), "https://reddit.com/r/test", "suffix {suffix}"); + } + } + + #[test] + fn traverse_absorbs_post_title_slug() { + let p = test_parts( + "reddit.com", + &["r", "rust", "comments", "aaa", "my_great_post_title"], + &[], + ); + assert_eq!(terminal(&p), Some("reddit_post")); + assert_eq!(canon(&p), "https://reddit.com/r/rust/comments/aaa"); + } + + #[test] + fn traverse_unknown_segment_falls_back_to_generic() { + let p = test_parts("reddit.com", &["r", "rust", "wiki", "faq"], &[]); + assert_eq!(terminal(&p), None); + assert_eq!(canon(&p), "https://reddit.com/r/rust/wiki/faq"); + } + + #[test] + fn traverse_youtube_watch_requires_v_in_query() { + let p = test_parts("youtube.com", &["watch"], &[("v", "xyz")]); + assert_eq!(terminal(&p), Some("youtube_watch")); + } + + #[test] + fn traverse_youtu_be_captures_vid_id() { + let p = test_parts("youtu.be", &["dQw4w9WgXcQ"], &[]); + assert_eq!(terminal(&p), Some("youtu_be_video")); + assert_eq!(vars(&p).get("vid_id").map(String::as_str), Some("dQw4w9WgXcQ")); + } + + #[test] + fn traverse_shorts_sets_video_id_var() { + let p = test_parts("youtube.com", &["shorts", "abc99"], &[]); + assert_eq!(terminal(&p), Some("youtube_watch")); + assert_eq!(vars(&p).get("video_id").map(String::as_str), Some("abc99")); + } + + #[test] + fn reddit_domain_canonical() { + let p = test_parts("reddit.com", &[], &[]); + assert_eq!(canon(&p), "https://reddit.com"); + } + + #[test] + fn reddit_r_hub_canonical() { + let p = test_parts("reddit.com", &["r"], &[]); + assert_eq!(terminal(&p), Some("reddit_r_hub")); + assert_eq!(canon(&p), "https://reddit.com/r"); + } + + #[test] + fn reddit_subreddit_lowercases_name() { + let p = test_parts("reddit.com", &["r", "AmITheAsshole"], &[]); + assert_eq!(canon(&p), "https://reddit.com/r/amitheasshole"); + } + + #[test] + fn reddit_host_aliases_old_new_www() { + for host in ["old.reddit.com", "new.reddit.com", "www.reddit.com"] { + let p = test_parts(host, &["r", "rust"], &[]); + assert_eq!(canon(&p), "https://reddit.com/r/rust", "host {host}"); + } + } + + #[test] + fn reddit_post_strips_slug_and_query() { + let p = test_parts( + "old.reddit.com", + &["r", "Rust", "comments", "1abc", "title_slug_here"], + &[("sort", "new")], + ); + assert_eq!(canon(&p), "https://reddit.com/r/rust/comments/1abc"); + } + + #[test] + fn reddit_post_multiple_slugs_absorbed() { + let p = test_parts( + "reddit.com", + &["r", "x", "comments", "id1", "slug1", "extra"], + &[], + ); + assert_eq!(canon(&p), "https://reddit.com/r/x/comments/id1"); + } + + #[test] + fn reddit_listing_with_query_only() { + let p = test_parts("www.reddit.com", &["r", "programming"], &[("sort", "top")]); + assert_eq!(canon(&p), "https://reddit.com/r/programming"); + } + + #[test] + fn reddit_subreddit_breadcrumbs_include_r_hub() { + let p = test_parts("reddit.com", &["r", "movies"], &[]); + assert_eq!( + crumbs(&p), + vec![ + "https://reddit.com", + "https://reddit.com/r", + "https://reddit.com/r/movies" + ] + ); + } + + #[test] + fn reddit_post_breadcrumbs_skip_comments_node() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "1trnvdl"], &[]); + let c = crumbs(&p); + assert!(!c.iter().any(|u| u.ends_with("/comments"))); + assert_eq!( + c.last().map(String::as_str), + Some("https://reddit.com/r/aww/comments/1trnvdl") + ); + assert!(c.contains(&"https://reddit.com/r/aww".to_string())); + } + + #[test] + fn reddit_post_parent_is_subreddit_not_comments() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "1trnvdl"], &[]); + let c = crumbs(&p); + let parent = c.get(c.len() - 2).unwrap(); + assert_eq!(parent, "https://reddit.com/r/aww"); + } + + #[test] + fn reddit_domain_parent_is_none_in_breadcrumb_chain() { + let p = test_parts("reddit.com", &[], &[]); + assert_eq!(crumbs(&p), vec!["https://reddit.com"]); + } + + #[test] + fn youtube_watch_canonical_uses_v_only() { + let p = test_parts("youtube.com", &["watch"], &[("v", "abc"), ("t", "99")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=abc"); + } + + #[test] + fn youtube_query_order_independent() { + let a = test_parts("youtube.com", &["watch"], &[("v", "abc"), ("t", "4")]); + let b = test_parts("youtube.com", &["watch"], &[("t", "4"), ("v", "abc")]); + assert_eq!(canon(&a), canon(&b)); + } + + #[test] + fn youtube_host_aliases() { + for host in ["www.youtube.com", "m.youtube.com"] { + let p = test_parts(host, &["watch"], &[("v", "x")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=x", "host {host}"); + } + } + + #[test] + fn youtube_shorts_canonical_matches_watch() { + let shorts = test_parts("youtube.com", &["shorts", "vid123"], &[]); + let watch = test_parts("youtube.com", &["watch"], &[("v", "vid123")]); + assert_eq!(canon(&shorts), canon(&watch)); + assert_eq!(canon(&shorts), "https://youtube.com/watch?v=vid123"); + } + + #[test] + fn youtu_be_matches_youtube_watch() { + let be = test_parts("youtu.be", &["dQw4w9WgXcQ"], &[]); + let watch = test_parts("youtube.com", &["watch"], &[("v", "dQw4w9WgXcQ")]); + assert_eq!(canon(&be), canon(&watch)); + } + + #[test] + fn youtube_breadcrumbs_domain_then_watch() { + let p = test_parts("youtube.com", &["watch"], &[("v", "abc")]); + assert_eq!( + crumbs(&p), + vec!["https://youtube.com", "https://youtube.com/watch?v=abc"] + ); + } + + #[test] + fn youtu_be_breadcrumbs_include_youtube_domain() { + let p = test_parts("youtu.be", &["abc"], &[]); + let c = crumbs(&p); + assert_eq!(c.first().map(String::as_str), Some("https://youtube.com")); + assert_eq!( + c.last().map(String::as_str), + Some("https://youtube.com/watch?v=abc") + ); + } + + #[test] + fn parsed_urls_match_hand_built_parts() { + let raw = "https://www.reddit.com/r/rust/comments/aaa/title/?utm=x"; + let parsed = UrlParts::parse(raw).unwrap(); + let hand = test_parts( + "www.reddit.com", + &["r", "rust", "comments", "aaa", "title"], + &[("utm", "x")], + ); + assert_eq!(canon(&parsed), canon(&hand)); + } + + #[test] + fn equivalence_cluster_youtube_formats() { + let urls = [ + "https://youtu.be/abc123", + "https://www.youtube.com/watch?v=abc123", + "https://youtube.com/watch?v=abc123&t=1", + "https://m.youtube.com/watch?t=1&v=abc123", + ]; + let canonical: Vec<_> = urls + .iter() + .map(|u| canon(&UrlParts::parse(u).unwrap())) + .collect(); + assert!(canonical.iter().all(|c| *c == "https://youtube.com/watch?v=abc123")); + } + + #[test] + fn equivalence_cluster_reddit_post_formats() { + let urls = [ + "https://old.reddit.com/r/Rust/comments/aaa/slug/", + "reddit.com/r/rust/comments/aaa/other_slug", + "https://reddit.com/r/RUST/comments/aaa", + ]; + let canonical: Vec<_> = urls + .iter() + .map(|u| canon(&UrlParts::parse(u).unwrap())) + .collect(); + assert!( + canonical + .iter() + .all(|c| *c == "https://reddit.com/r/rust/comments/aaa") + ); + } + + #[test] + fn graph_nodes_all_have_valid_parent_links() { + let g = build_graph(); + for (id, node) in &g.nodes { + if let Some(parent) = node.parent { + assert!(g.nodes.contains_key(parent), "node {id} parent {parent}"); + } + } + } + + #[test] + fn graph_terminal_canonical_always_succeeds_for_reddit_paths() { + let cases: &[(&[&str], &str)] = &[ + (&["r", "rust"], "https://reddit.com/r/rust"), + ( + &["r", "rust", "comments", "x"], + "https://reddit.com/r/rust/comments/x", + ), + ]; + for (segs, want) in cases { + let p = test_parts("reddit.com", segs, &[]); + assert_eq!(canon(&p), *want); + } + } + + #[test] + fn breadcrumb_parent_walk_matches_parent_url_semantics() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "id1"], &[]); + let c = crumbs(&p); + assert_eq!(c.len(), 4); + assert_eq!( + c.get(c.len() - 2).map(String::as_str), + Some("https://reddit.com/r/aww") + ); + } + + #[test] + fn youtube_watch_without_v_falls_back_to_generic() { + let p = test_parts("youtube.com", &["watch"], &[]); + assert_eq!(terminal(&p), Some("youtube_watch")); + assert_eq!(canon(&p), "https://youtube.com/watch"); + } + + #[test] + fn reddit_only_comments_path_stops_at_gate() { + let p = test_parts("reddit.com", &["r", "rust", "comments"], &[]); + assert_eq!(terminal(&p), Some("reddit_comments_gate")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn generic_deep_path_many_segments() { + let segs: Vec<&str> = (0..10) + .map(|i| match i { + 0 => "a", + 1 => "b", + 2 => "c", + 3 => "d", + 4 => "e", + 5 => "f", + 6 => "g", + 7 => "h", + 8 => "i", + _ => "j", + }) + .collect(); + let p = test_parts("site.com", &segs, &[]); + assert_eq!(crumbs(&p).len(), 11); + } + + #[test] + fn traverse_literal_r_required_for_subreddit() { + let p = test_parts("reddit.com", &["rust"], &[]); + assert_eq!(terminal(&p), None); + } + + #[test] + fn http_scheme_upgraded_via_generic_fallback_host() { + let parsed = UrlParts::parse("http://example.com/page").unwrap(); + assert_eq!(canon(&parsed), "https://example.com/page"); + } + + #[test] + fn each_graph_node_canonical_is_invokable() { + let g = build_graph(); + let empty = Context::default(); + for (id, node) in &g.nodes { + let _ = (node.canonical)(&empty); + let _ = id; + } + } + + #[test] + fn reddit_double_listing_suffix_both_absorbed() { + let p = test_parts("reddit.com", &["r", "rust", "hot", "new"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn youtu_be_empty_path_stays_at_entry() { + let p = test_parts("youtu.be", &[], &[]); + assert_eq!(terminal(&p), Some("youtu_be_entry")); + } +} diff --git a/server/src/url_rules/graph_builder.rs b/server/src/url_rules/graph_builder.rs new file mode 100644 index 0000000000000000000000000000000000000000..243204ad5ca514fff459057956080cacb5bf38e2 --- /dev/null +++ b/server/src/url_rules/graph_builder.rs @@ -0,0 +1,73 @@ +//! Declarative construction of the URL graph with build-time link validation. + +use std::collections::HashMap; + +use super::graph::{CanonicalFn, Edge, EdgePattern, Graph, Node}; + +pub struct GraphBuilder { + nodes: HashMap<&'static str, Node>, + current: Option<&'static str>, +} + +impl GraphBuilder { + pub fn new() -> Self { + Self { + nodes: HashMap::new(), + current: None, + } + } + + pub fn node(mut self, id: &'static str) -> Self { + self.nodes.entry(id).or_insert_with(Node::empty); + self.current = Some(id); + self + } + + pub fn canonical(mut self, f: CanonicalFn) -> Self { + let id = self.current.expect("canonical() without node()"); + self.nodes.get_mut(id).expect("node missing").canonical = f; + self + } + + pub fn parent(mut self, parent_id: &'static str) -> Self { + let id = self.current.expect("parent() without node()"); + self.nodes.get_mut(id).expect("node missing").parent = Some(parent_id); + self + } + + pub fn edge(mut self, pattern: EdgePattern, target: &'static str) -> Self { + let id = self.current.expect("edge() without node()"); + self.nodes + .get_mut(id) + .expect("node missing") + .edges + .push(Edge { pattern, target }); + self + } + + pub fn build(self) -> Graph { + for (id, node) in &self.nodes { + if let Some(parent) = node.parent { + assert!( + self.nodes.contains_key(parent), + "node {id}: parent {parent} does not exist" + ); + } + for edge in &node.edges { + if !matches!( + edge.pattern, + EdgePattern::AbsorbAny | EdgePattern::AbsorbIf(_) + ) { + assert!( + self.nodes.contains_key(edge.target), + "node {id}: edge target {} does not exist", + edge.target + ); + } + } + } + Graph { + nodes: self.nodes, + } + } +} diff --git a/server/src/url_rules/mod.rs b/server/src/url_rules/mod.rs index 9e1445346ce77a49dd6a7e7713bf9c57aef353cc..ba4ac662acc7bd4d50ee34613eb7eb6fccbfb17b 100644 --- a/server/src/url_rules/mod.rs +++ b/server/src/url_rules/mod.rs @@ -1,6 +1,7 @@ //! URL canonicalization and hierarchy via a semantic graph (DFA + generic fallback). mod graph; +mod graph_builder; mod parse; mod registry; diff --git a/server/src/url_rules/parse.rs b/server/src/url_rules/parse.rs new file mode 100644 index 0000000000000000000000000000000000000000..19d0c82feb718817168b8b445fcbfa39cf6aa6ba --- /dev/null +++ b/server/src/url_rules/parse.rs @@ -0,0 +1,169 @@ +//! Parse raw strings into host, path segments, and query (order-independent). + +use std::collections::HashMap; + +use url::Url; + +#[derive(Debug, Clone)] +pub struct UrlParts { + pub scheme: String, + pub host: String, + pub path_segments: Vec, + pub query: HashMap, +} + +impl UrlParts { + pub fn parse(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + let with_scheme = if trimmed.contains("://") { + trimmed.to_string() + } else if trimmed.starts_with("r/") || trimmed.starts_with("/r/") { + let rest = trimmed.trim_start_matches('/').trim_start_matches("r/"); + format!("https://reddit.com/r/{rest}") + } else if trimmed.contains('.') && !trimmed.starts_with('/') { + format!("https://{trimmed}") + } else { + trimmed.to_string() + }; + + let url = Url::parse(&with_scheme).ok()?; + let host = url.host_str()?.to_string(); + let path_segments: Vec = url + .path_segments() + .map(|segs| segs.filter(|s| !s.is_empty()).map(str::to_string).collect()) + .unwrap_or_default(); + + let mut query = HashMap::new(); + for (k, v) in url.query_pairs() { + query.insert(k.into_owned(), v.into_owned()); + } + + Some(Self { + scheme: url.scheme().to_string(), + path_segments, + query, + host, + }) + } + + /// Host normalized for graph entry matching (lowercase, aliases). + pub fn match_host(&self) -> String { + normalize_match_host(&self.host) + } +} + +pub fn normalize_match_host(host: &str) -> String { + let h = host + .strip_prefix("www.") + .unwrap_or(host) + .to_ascii_lowercase(); + match h.as_str() { + "old.reddit.com" | "new.reddit.com" => "reddit.com".to_string(), + "m.youtube.com" => "youtube.com".to_string(), + _ => h, + } +} + +pub fn strip_tracking_query(query: &mut HashMap) { + query.retain(|k, _| { + let lower = k.to_ascii_lowercase(); + !(lower.starts_with("utm_") + || matches!( + lower.as_str(), + "fbclid" | "gclid" | "ref" | "ref_src" | "ref_source" | "mc_cid" | "mc_eid" + )) + }); +} + +#[cfg(test)] +pub(crate) fn test_parts(host: &str, segs: &[&str], query: &[(&str, &str)]) -> UrlParts { + UrlParts { + scheme: "https".to_string(), + host: host.to_string(), + path_segments: segs.iter().map(|s| (*s).to_string()).collect(), + query: query + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_full_url_splits_host_path_query() { + let p = UrlParts::parse("https://www.youtube.com/watch?v=abc&t=4").unwrap(); + assert_eq!(p.host, "www.youtube.com"); + assert_eq!(p.path_segments, vec!["watch"]); + assert_eq!(p.query.get("v").map(String::as_str), Some("abc")); + assert_eq!(p.query.get("t").map(String::as_str), Some("4")); + } + + #[test] + fn parse_r_shortcut_expands_to_reddit() { + let p = UrlParts::parse("r/rust").unwrap(); + assert_eq!(p.match_host(), "reddit.com"); + assert_eq!(p.path_segments, vec!["r", "rust"]); + } + + #[test] + fn parse_slash_r_shortcut() { + let p = UrlParts::parse("/r/aww").unwrap(); + assert_eq!(p.path_segments, vec!["r", "aww"]); + } + + #[test] + fn parse_schemeless_host_path() { + let p = UrlParts::parse("reddit.com/r/rust/comments/aaa/slug").unwrap(); + assert_eq!(p.match_host(), "reddit.com"); + assert_eq!( + p.path_segments, + vec!["r", "rust", "comments", "aaa", "slug"] + ); + } + + #[test] + fn parse_empty_returns_none() { + assert!(UrlParts::parse("").is_none()); + assert!(UrlParts::parse(" ").is_none()); + } + + #[test] + fn normalize_match_host_reddit_aliases() { + assert_eq!(normalize_match_host("old.reddit.com"), "reddit.com"); + assert_eq!(normalize_match_host("NEW.reddit.com"), "reddit.com"); + assert_eq!(normalize_match_host("www.reddit.com"), "reddit.com"); + } + + #[test] + fn normalize_match_host_youtube_aliases() { + assert_eq!(normalize_match_host("m.youtube.com"), "youtube.com"); + assert_eq!(normalize_match_host("www.youtube.com"), "youtube.com"); + } + + #[test] + fn strip_tracking_query_removes_known_params() { + let mut q = HashMap::from([ + ("v".into(), "1".into()), + ("utm_source".into(), "x".into()), + ("fbclid".into(), "y".into()), + ("ref".into(), "z".into()), + ]); + strip_tracking_query(&mut q); + assert_eq!(q.len(), 1); + assert_eq!(q.get("v").map(String::as_str), Some("1")); + } + + #[test] + fn strip_tracking_query_utm_prefix() { + let mut q = HashMap::from([("utm_campaign".into(), "email".into())]); + strip_tracking_query(&mut q); + assert!(q.is_empty()); + } +} diff --git a/server/src/url_rules/registry_tests.rs b/server/src/url_rules/registry_tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..7b76b550f808f590e011469fdae02adf603b43d2 --- /dev/null +++ b/server/src/url_rules/registry_tests.rs @@ -0,0 +1,195 @@ +//! End-to-end tests for the public registry API (`canonicalize_raw`, breadcrumbs, parent). + +use super::registry::{ + canonicalize_raw, looks_like_url, navigable_breadcrumbs, parent_url, resolve_id, +}; + +fn canon(raw: &str) -> String { + canonicalize_raw(raw).unwrap().canonical +} + +#[test] +fn looks_like_url_positive_cases() { + for raw in [ + "https://reddit.com/r/rust", + "r/rust", + "/r/aww", + "reddit.com/r/x", + "www.example.com/path", + "youtu.be/abc", + ] { + assert!(looks_like_url(raw), "{raw}"); + } +} + +#[test] +fn looks_like_url_negative_cases() { + for raw in ["alpha", "beta", "", "hello world", "no-dots"] { + assert!(!looks_like_url(raw), "{raw}"); + } +} + +#[test] +fn resolve_id_matches_canonicalize_raw() { + let raw = "https://youtu.be/xyz"; + assert_eq!( + resolve_id(raw).as_deref(), + Some(canon(raw).as_str()) + ); +} + +#[test] +fn canonicalize_empty_returns_none() { + assert!(canonicalize_raw("").is_none()); +} + +#[test] +fn alias_when_slug_stripped() { + let r = canonicalize_raw( + "https://reddit.com/r/rust/comments/aaa/very_long_title_slug", + ) + .unwrap(); + assert_eq!(r.canonical, "https://reddit.com/r/rust/comments/aaa"); + assert!(r.alias_of.is_some()); +} + +#[test] +fn alias_none_when_already_canonical() { + let raw = "https://reddit.com/r/rust"; + let r = canonicalize_raw(raw).unwrap(); + assert_eq!(r.canonical, raw); + assert!(r.alias_of.is_none()); +} + +#[test] +fn parent_url_subreddit_under_r_hub() { + assert_eq!( + parent_url("https://reddit.com/r/movies").as_deref(), + Some("https://reddit.com/r") + ); +} + +#[test] +fn parent_url_domain_has_none() { + assert_eq!(parent_url("https://reddit.com").as_deref(), None); +} + +#[test] +fn parent_url_generic_site() { + assert_eq!( + parent_url("https://example.com/a/b").as_deref(), + Some("https://example.com/a") + ); +} + +#[test] +fn breadcrumbs_from_canonical_string_roundtrip() { + let id = "https://reddit.com/r/golang/comments/abc123"; + let crumbs = navigable_breadcrumbs(id); + assert_eq!(crumbs.last().map(String::as_str), Some(id)); +} + +// --- Table: Reddit raw URLs → canonical --- + +#[test] +fn reddit_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ("r/rust", "https://reddit.com/r/rust"), + ("/r/aww", "https://reddit.com/r/aww"), + ("https://reddit.com/r/rust", "https://reddit.com/r/rust"), + ( + "https://www.reddit.com/r/programming/new", + "https://reddit.com/r/programming", + ), + ( + "https://old.reddit.com/r/test/comments/xyz/slug/", + "https://reddit.com/r/test/comments/xyz", + ), + ( + "reddit.com/r/Movies/comments/abc/Title_Case_Slug", + "https://reddit.com/r/movies/comments/abc", + ), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Table: YouTube raw URLs → canonical --- + +#[test] +fn youtube_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ( + "https://youtube.com/watch?v=abc", + "https://youtube.com/watch?v=abc", + ), + ( + "https://www.youtube.com/watch?v=abc&t=1&feature=share", + "https://youtube.com/watch?v=abc", + ), + ("https://youtu.be/abc", "https://youtube.com/watch?v=abc"), + ( + "https://youtube.com/shorts/abc", + "https://youtube.com/watch?v=abc", + ), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Table: generic sites --- + +#[test] +fn generic_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ( + "https://news.ycombinator.com/item?id=38472", + "https://news.ycombinator.com/item?id=38472", + ), + ( + "https://www.github.com/rust-lang/rust/issues/1?utm_source=x", + "https://github.com/rust-lang/rust/issues/1", + ), + ("https://example.com", "https://example.com"), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Phantom /comments/ regression (sorter2-specific) --- + +#[test] +fn phantom_comments_not_in_breadcrumbs_for_post() { + let crumbs = navigable_breadcrumbs("https://reddit.com/r/rust/comments/aaa"); + assert!(!crumbs.iter().any(|c| c.ends_with("/comments"))); +} + +#[test] +fn phantom_comments_not_sibling_of_subreddit_in_breadcrumb_chain() { + let crumbs = navigable_breadcrumbs("https://reddit.com/r/rust/comments/aaa"); + let subs: Vec<_> = crumbs + .iter() + .filter(|c| c.contains("/r/rust") && !c.contains("/comments/")) + .collect(); + assert_eq!(subs, vec!["https://reddit.com/r/rust"]); +} + +// --- Distinct items must stay distinct --- + +#[test] +fn different_posts_different_canonical() { + let a = canon("https://reddit.com/r/rust/comments/aaa"); + let b = canon("https://reddit.com/r/rust/comments/bbb"); + assert_ne!(a, b); +} + +#[test] +fn different_subreddits_different_canonical() { + assert_ne!( + canon("https://reddit.com/r/rust"), + canon("https://reddit.com/r/golang") + ); +}