You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [00be3a29] invite system Side A — unified diff (full patch): diff --git a/bb.edn b/bb.edn index 50be8232847e672b3f273a2fb25ddd1d12adb7e2..818f850765370d12d58f239e287764fc3649d78b 100644 --- a/bb.edn +++ b/bb.edn @@ -47,14 +47,16 @@ "RUST_LOG" "info"})})))} test - {:doc "Full test suite: integration + auth + grants" + {:doc "Full test suite: integration + auth + grants + invites" :requires ([test.integration :as integration] [test.auth :as auth] - [test.grants :as grants]) + [test.grants :as grants] + [test.invites :as invites]) :task (do (integration/integration) (auth/auth-test) - (grants/grants-test))} + (grants/grants-test) + (invites/invites-test))} perf {:doc "Performance test: concurrent HTTP requests to detect blocking I/O" diff --git a/cli/src/main.rs b/cli/src/main.rs index e5833b0b93d8e667b94c574ba2b0f8cb758ff3df..8eda9f485bd1f7392f1e34be27176c21e20354eb 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -105,6 +105,23 @@ enum ScopedCmd { #[arg(long)] json: bool, }, + + /// Mint a shareable invite link (24h TTL, in-memory until redeemed). Requires Manage on the room. + InviteLink { + /// Comma-separated: view, post, vote, add_item, manage + #[arg(long = "caps", value_delimiter = ',')] + caps: Vec, + #[arg(long, default_value_t = 1)] + uses: usize, + #[arg(long)] + json: bool, + }, + + /// List principals granted access in this room (requires View or Manage) + Audit { + #[arg(long)] + json: bool, + }, } #[derive(Subcommand, Debug)] @@ -514,17 +531,35 @@ fn print_thread(resp: &ThreadDetailResponse) { .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as i64; - if resp.total > resp.posts.len() { - let end = resp.offset + resp.posts.len(); - eprintln!("# showing {}-{} of {} posts (--offset N --limit N to paginate)", resp.offset, end.saturating_sub(1), resp.total); + if resp.total > resp.items.len() { + let end = resp.offset + resp.items.len(); + eprintln!( + "# showing {}-{} of {} rows (--offset N --limit N to paginate)", + resp.offset, + end.saturating_sub(1), + resp.total + ); } - for (i, post) in resp.posts.iter().enumerate() { - let timeago = slug_types::timeago::timeago_compact(now_ms, post.ts); - let body = &post.body.trim(); - println!("", post.index, timeago); - println!("{}", body); - println!(""); - if i + 1 < resp.posts.len() { + for (i, item) in resp.items.iter().enumerate() { + match item { + ThreadItem::Post { + index, + ts, + body, + .. + } => { + let timeago = slug_types::timeago::timeago_compact(now_ms, *ts); + let body = body.trim(); + println!("", index, timeago); + println!("{}", body); + println!(""); + } + ThreadItem::System { ts, text } => { + let timeago = slug_types::timeago::timeago_compact(now_ms, *ts); + println!("{}", timeago, text.trim()); + } + } + if i + 1 < resp.items.len() { println!(); println!(); } @@ -1036,6 +1071,95 @@ async fn run_scoped(base: &str, room: &str, sub: ScopedCmd) -> Result<()> { } } }, + ScopedCmd::InviteLink { caps, uses, json } => { + let caps: Vec = caps + .into_iter() + .flat_map(|s| { + s.split(',') + .map(|p| p.trim().to_lowercase()) + .filter(|p| !p.is_empty()) + .collect::>() + }) + .collect(); + if caps.is_empty() { + return Err(anyhow!("--caps is required (e.g. --caps view,post,vote)")); + } + let bearer = effective_bearer().ok_or_else(|| { + anyhow!( + "no bearer token: run `slugsocial identity start --rig --model ` \ + then `slugsocial identity poll `, or set SLUG_BEARER_TOKEN / ~/.config/slugsocial/token" + ) + })?; + let batch = send_rpc( + &client, + base, + Some(&bearer), + vec![RpcCommand::RoomMintInvite { + room: room.to_string(), + capabilities: caps, + max_uses: uses, + }], + ) + .await?; + match rpc_line_ok(&batch.results[0])? { + RpcResult::RoomInviteMinted { + invite_url, + expires_at_ms, + max_uses, + } => { + if json { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "invite_url": invite_url, + "expires_at_ms": expires_at_ms, + "max_uses": max_uses, + }))? + ); + } else { + println!("{invite_url}"); + println!("(Expires in 24 hours. Max uses: {max_uses})"); + } + } + _ => return Err(anyhow!("unexpected RPC result")), + } + } + ScopedCmd::Audit { json } => { + let bearer = effective_bearer().ok_or_else(|| { + anyhow!( + "no bearer token: run `slugsocial identity start --rig --model ` \ + then `slugsocial identity poll `, or set SLUG_BEARER_TOKEN / ~/.config/slugsocial/token" + ) + })?; + let batch = send_rpc( + &client, + base, + Some(&bearer), + vec![RpcCommand::RoomAudit { + room: room.to_string(), + }], + ) + .await?; + match rpc_line_ok(&batch.results[0])? { + RpcResult::RoomAudit(resp) => { + if json { + println!("{}", serde_json::to_string_pretty(&resp)?); + } else { + println!("room {}", resp.room); + if resp.grants.is_empty() { + println!("(no grants recorded)"); + } else { + let w_user = resp.grants.iter().map(|g| g.username.len()).max().unwrap_or(0); + for g in &resp.grants { + let caps = g.capabilities.join(", "); + println!("{: return Err(anyhow!("unexpected RPC result")), + } + } ScopedCmd::Check { file, json } => { let mut text = String::new(); match file { diff --git a/server/src/api/auth.rs b/server/src/api/auth.rs index 995ce4a61d29b024c399c656134f541ecfd880cf..b45ba39419c84af8bf2333fc9b7d47e98525c45f 100644 --- a/server/src/api/auth.rs +++ b/server/src/api/auth.rs @@ -12,12 +12,61 @@ use tokio::sync::RwLock; use crate::{ api::helpers::{api_error, now_ms, sha256_hex}, - events::{Event, TokenIssued, UserRegistered}, + events::{Event, GrantAdded, TokenIssued, UserRegistered}, identity::{parse_agent, parse_username}, html::{auth_complete_page, auth_signed_in_fragment, choose_username_error_fragment, choose_username_page}, state::{AppState, PendingSession}, }; +/// Delegate id for browser users who land via `/join/inv_…` (no CLI agent). +const INVITE_BROWSER_AGENT: &str = "00000000-0000-0000-0000-000000000000:invite:web/join"; + +async fn apply_invite_redemption(state: &AppState, invite_token: &str, grantee_username: &str) -> Result<(), String> { + let now = now_ms(); + let ga = { + let mut invites = state.invites.write().await; + let Some(inv) = invites.get_mut(invite_token) else { + return Err("invite not found".into()); + }; + if now > inv.expires_at_ms { + invites.remove(invite_token); + return Err("invite expired".into()); + } + if inv.current_uses >= inv.max_uses { + return Err("invite exhausted".into()); + } + inv.current_uses += 1; + Event::GrantAdded(GrantAdded { + ts: now, + room_id: inv.room_id.clone(), + username: grantee_username.to_string(), + capabilities: inv.capabilities.clone(), + granted_by: inv.inviter.clone(), + }) + }; + + match state.event_log.append(&ga).await { + Ok(()) => { + let mut reduced = state.reduced.write().await; + reduced.apply_event(ga); + let mut invites = state.invites.write().await; + if let Some(inv) = invites.get(invite_token) { + if inv.current_uses >= inv.max_uses { + invites.remove(invite_token); + } + } + Ok(()) + } + Err(e) => { + let mut invites = state.invites.write().await; + if let Some(inv) = invites.get_mut(invite_token) { + inv.current_uses = inv.current_uses.saturating_sub(1); + } + Err(format!("{e}")) + } + } +} + fn pending_sessions(state: &AppState) -> Arc>> { state.pending_sessions.clone() } @@ -115,6 +164,42 @@ pub struct AuthLoginQuery { pub session: String, } +pub async fn get_join_invite(Path(token): Path, State(state): State) -> impl IntoResponse { + let token = token.trim().to_string(); + if token.is_empty() { + return api_error(StatusCode::NOT_FOUND, "invite invalid or expired", None).into_response(); + } + let now = now_ms(); + let valid = { + let invites = state.invites.read().await; + match invites.get(&token) { + None => false, + Some(inv) => now <= inv.expires_at_ms && inv.current_uses < inv.max_uses, + } + }; + if !valid { + return api_error(StatusCode::NOT_FOUND, "invite invalid or expired", None).into_response(); + } + + let session = format!("p_{}", uuid::Uuid::new_v4().simple()); + let s = PendingSession { + agent: INVITE_BROWSER_AGENT.to_string(), + created_ts: now_ms(), + provider: None, + provider_id: None, + redeem_invite: Some(token), + complete: None, + }; + state.pending_sessions.write().await.insert(session.clone(), s); + + let public_url = std::env::var("SLUG_PUBLIC_URL").unwrap_or_else(|_| "http://127.0.0.1:8080".to_string()); + Redirect::temporary(&format!( + "{public_url}/auth/login?session={}", + urlencoding::encode(&session) + )) + .into_response() +} + pub async fn get_auth_login(Query(q): Query, State(state): State) -> impl IntoResponse { // Redirect to Google auth endpoint. let sessions = pending_sessions(&state); @@ -205,6 +290,7 @@ pub async fn get_auth_callback(Query(q): Query, State(state): s.provider = Some("google".to_string()); s.provider_id = Some(sub.clone()); if let Some(username) = existing { + let invite_tok = s.redeem_invite.clone(); let (bearer, token_event) = issue_token_for_user(&username); // append token event let ev = Event::TokenIssued(token_event); @@ -215,6 +301,11 @@ pub async fn get_auth_callback(Query(q): Query, State(state): let mut reduced = reduced_arc.write().await; reduced.apply_event(ev); } + if let Some(tok) = invite_tok { + if let Err(e) = apply_invite_redemption(&state, &tok, &username).await { + tracing::warn!(error = %e, "invite redemption skipped after oauth"); + } + } s.complete = Some((username, bearer)); return Redirect::temporary(&format!("{public_url}/auth/complete")).into_response(); } @@ -310,6 +401,16 @@ pub async fn post_choose_username( reduced.apply_event(ti_ev.clone()); } + // Redeem invite (if any) before marking the session complete. + if let Some(tok) = { + let sessions_read = sessions.read().await; + sessions_read.get(&form.session).and_then(|s| s.redeem_invite.clone()) + } { + if let Err(e) = apply_invite_redemption(&state, &tok, &canon_user).await { + tracing::warn!(error = %e, "invite redemption skipped after registration"); + } + } + // Mark complete for polling. { let mut sessions_write = sessions.write().await; @@ -339,6 +440,7 @@ pub async fn post_pending_session( created_ts: now_ms(), provider: None, provider_id: None, + redeem_invite: None, complete: None, }; let sessions = pending_sessions(&state); diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index a1ea432932cdca0e42dc7377f0a25075fd5644c0..cb031aecebad1107dfa2898a98fb6084b28ba6e9 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -4,6 +4,7 @@ mod rpc; mod validate; pub use auth::{ + get_join_invite, get_pending_session, get_whoami, post_pending_session, diff --git a/server/src/api/rpc.rs b/server/src/api/rpc.rs index 11a76ed7a02aae588edadb7a87a43660f5669d9e..f6bbc3df71909a2da7403cd46fe4ea6ca130c692 100644 --- a/server/src/api/rpc.rs +++ b/server/src/api/rpc.rs @@ -13,12 +13,14 @@ use slug_types::*; use crate::{ canonical_path::{canonicalize_item, canonicalize_tag}, dsl, - events::{AgentBound, Event, GrantAdded, Ingest, RoomCreated, ThreadCapability, ThreadVisibility}, + events::{ + AgentBound, Event, GrantAdded, Ingest, RoomCreated, ThreadCapability, ThreadVisibility, + }, identity::{parse_agent, parse_username}, path_types::CanonicalItemUrl, ranking::{connected_components_from_voted_pairs, ranked_items_subset}, reducer::{scope_from_room_wire, ReducerState, ScopeId}, - state::AppState, + state::{AppState, InviteState}, }; use super::auth::verify_bearer_principal; @@ -142,6 +144,27 @@ fn gen_short_id() -> String { (0..7).map(|_| ALPHABET[rng.gen_range(0..ALPHABET.len())] as char).collect() } +fn gen_invite_token() -> String { + use rand::Rng; + const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + let mut rng = rand::thread_rng(); + let tail: String = (0..16).map(|_| ALPHABET[rng.gen_range(0..ALPHABET.len())] as char).collect(); + format!("inv_{tail}") +} + +const INVITE_TTL_MS: i64 = 86_400_000; + +fn capability_wire(c: ThreadCapability) -> String { + match c { + ThreadCapability::View => "view", + ThreadCapability::Post => "post", + ThreadCapability::Vote => "vote", + ThreadCapability::AddItem => "add_item", + ThreadCapability::Manage => "manage", + } + .to_string() +} + fn build_rank_response_for_content( content: &crate::reducer::ContentState, parent: Option<&str>, @@ -512,7 +535,7 @@ fn rpc_forum_thread_detail( None => Err(("post not found".into(), None)), Some((idx, ing)) => Ok(ThreadDetailResponse { thread: format!("#{}", tag), - posts: vec![PostRow { + items: vec![ThreadItem::Post { id: ing.id.clone(), index: idx, ts: ing.ts, @@ -543,7 +566,7 @@ fn rpc_forum_thread_detail( let total = filtered.len(); const MAX_BODY: usize = 2000; - let posts: Vec = filtered + let items: Vec = filtered .into_iter() .skip(offset) .take(limit) @@ -553,7 +576,7 @@ fn rpc_forum_thread_detail( } else { (ing.raw.clone(), false) }; - PostRow { + ThreadItem::Post { id: ing.id.clone(), index: idx, ts: ing.ts, @@ -566,7 +589,7 @@ fn rpc_forum_thread_detail( Ok(ThreadDetailResponse { thread: format!("#{}", tag), - posts, + items, total, offset, }) @@ -1007,7 +1030,7 @@ pub async fn handle_rpc_batch( RpcCommand::RoomGrant { room, username, - capability, + capabilities, } => { let principal = { let reduced = state.reduced.read().await; @@ -1022,6 +1045,8 @@ pub async fn handle_rpc_batch( }; if !can_manage { line_err("requires Manage capability", None) + } else if capabilities.is_empty() { + line_err("capabilities must not be empty", None) } else { match parse_username(&username) { Err(msg) => line_err("invalid username", Some(msg)), @@ -1033,14 +1058,18 @@ pub async fn handle_rpc_batch( if !user_exists { line_err(format!("user @{target} not found"), None) } else { - match parse_capability(&capability) { + let caps: Result, String> = capabilities + .iter() + .map(|c| parse_capability(c.trim())) + .collect(); + match caps { Err(msg) => line_err(msg, None), - Ok(cap) => { + Ok(caps) => { let ga_ev = Event::GrantAdded(GrantAdded { ts: now_ms(), room_id: room, username: target, - capabilities: vec![cap], + capabilities: caps, granted_by: principal, }); if let Err(e) = state.event_log.append(&ga_ev).await { @@ -1059,6 +1088,114 @@ pub async fn handle_rpc_batch( } } } + RpcCommand::RoomMintInvite { + room, + capabilities, + max_uses, + } => { + let principal = { + let reduced = state.reduced.read().await; + verify_bearer_principal(&headers, &*reduced) + }; + match principal { + Err((_, m)) => line_err(m, None), + Ok(principal) => { + let can_manage = { + let reduced = state.reduced.read().await; + reduced.user_has_cap(&room, &principal, ThreadCapability::Manage) + }; + if !can_manage { + line_err("requires Manage capability", None) + } else if capabilities.is_empty() { + line_err("capabilities must not be empty", None) + } else { + match capabilities + .iter() + .map(|c| parse_capability(c.trim())) + .collect::, String>>() + { + Err(msg) => line_err(msg, None), + Ok(caps) => { + let max_uses = max_uses.max(1).min(100_000); + let now = now_ms(); + let expires_at_ms = now + INVITE_TTL_MS; + let token = loop { + let t = gen_invite_token(); + let taken = { + let invites = state.invites.read().await; + invites.contains_key(&t) + }; + if !taken { + break t; + } + }; + let inv = InviteState { + room_id: room.clone(), + capabilities: caps, + expires_at_ms, + max_uses, + current_uses: 0, + inviter: principal, + }; + state.invites.write().await.insert(token.clone(), inv); + let public_url = std::env::var("SLUG_PUBLIC_URL") + .unwrap_or_else(|_| "http://127.0.0.1:8080".to_string()); + let invite_url = format!("{public_url}/join/{token}"); + line_ok(RpcResult::RoomInviteMinted { + invite_url, + expires_at_ms: Some(expires_at_ms), + max_uses, + }) + } + } + } + } + } + } + RpcCommand::RoomAudit { room } => { + let principal = { + let reduced = state.reduced.read().await; + verify_bearer_principal(&headers, &*reduced) + }; + match principal { + Err((_, m)) => line_err(m, None), + Ok(principal) => { + let reduced = state.reduced.read().await; + if !reduced.rooms.contains_key(&room) { + line_err("unknown room", None) + } else { + let can_audit = reduced.user_has_cap(&room, &principal, ThreadCapability::View) + || reduced.user_has_cap(&room, &principal, ThreadCapability::Manage); + if !can_audit { + line_err("requires View or Manage capability", None) + } else { + let grants: Vec = reduced + .grants + .get(&room) + .map(|m| { + let mut v: Vec = m + .iter() + .map(|(username, caps)| { + let mut c: Vec = + caps.iter().copied().map(capability_wire).collect(); + c.sort(); + RoomAuditEntry { + username: username.clone(), + capabilities: c, + } + }) + .collect(); + v.sort_by(|a, b| a.username.cmp(&b.username)); + v + }) + .unwrap_or_default(); + line_ok(RpcResult::RoomAudit(RoomAuditResponse { room, grants })) + } + } + } + } + } + RpcCommand::RoomRevoke { .. } => line_err("RoomRevoke is not implemented yet", None), RpcCommand::GetGlobalRank { room, limit, diff --git a/server/src/events.rs b/server/src/events.rs index 125005d89ade7b22c9b6d997f0ed81781df5e7ec..9e60f2c218f5c871a93e855e415dd984e3a017c5 100644 --- a/server/src/events.rs +++ b/server/src/events.rs @@ -26,6 +26,8 @@ pub enum Event { RoomCreated(RoomCreated), GrantAdded(GrantAdded), GrantRevoked(GrantRevoked), + InviteMinted(InviteMinted), + InviteRedeemed(InviteRedeemed), /// Ingest of a DSL+prose body. Identity and routing live in event metadata. Ingest(Ingest), } @@ -82,6 +84,25 @@ pub struct GrantRevoked { pub revoked_by: String, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct InviteMinted { + pub ts: i64, + pub token: String, + pub room_id: String, + pub capabilities: Vec, + pub inviter: String, + pub max_uses: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_ts_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct InviteRedeemed { + pub ts: i64, + pub token: String, + pub username: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Ingest { /// Unix timestamp in milliseconds. diff --git a/server/src/lib.rs b/server/src/lib.rs index d00b56d909e1fabd1d122d054dd02785e13e0bd8..bf2f73d4d0f26a357961e88c52c3b4f72626af81 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -27,6 +27,7 @@ pub fn create_app(state: AppState) -> Router { Router::new() .route("/healthz", axum::routing::get(|| async { "ok" })) .route("/static/:filename", axum::routing::get(crate::html::serve_theme_css)) + .route("/join/:token", axum::routing::get(api::get_join_invite)) .route("/auth/login", axum::routing::get(api::get_auth_login)) .route("/auth/callback", axum::routing::get(api::get_auth_callback)) .route("/auth/complete", axum::routing::get(api::get_auth_complete)) diff --git a/server/src/reducer.rs b/server/src/reducer.rs index 118cb19f4df4a083df2878ca7a1424f3d63de851..5190d0a7ee3d545a786a68fef483322aafdce7f9 100644 --- a/server/src/reducer.rs +++ b/server/src/reducer.rs @@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use serde::{Deserialize, Serialize}; use crate::canonical_path::canonicalize_tag; -use crate::events::{Event, Ingest, ThreadCapability}; +use crate::events::{Event, Ingest, ThreadCapability, ThreadVisibility}; use crate::path_types::CanonicalItemUrl; #[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] @@ -156,6 +156,41 @@ pub struct RoomState { pub visibility: crate::events::ThreadVisibility, } +/// Durable invite link state (from [`crate::events::InviteMinted`] / [`crate::events::InviteRedeemed`]). +#[derive(Debug, Clone)] +pub struct ActiveInviteState { + pub room_id: String, + pub capabilities: HashSet, + pub inviter: String, + pub uses_remaining: u32, + pub expires_ts_ms: Option, +} + +#[derive(Clone, Debug)] +pub enum RoomTimelineKind { + RoomCreated { + owner: String, + slug: String, + visibility: ThreadVisibility, + }, + GrantAdded { + username: String, + granted_by: String, + capabilities: Vec, + }, + GrantRevoked { + username: String, + revoked_by: String, + capabilities: Vec, + }, +} + +#[derive(Clone, Debug)] +pub struct RoomTimelineEntry { + pub ts: i64, + pub kind: RoomTimelineKind, +} + #[derive(Debug, Clone)] pub struct ForumThreadState { pub last_activity_ts: i64, @@ -210,6 +245,10 @@ pub struct ReducerState { pub ingests_ordered: Vec, /// room_id → username → capabilities pub grants: HashMap>>, + /// room_id → chronological room admin lines (for thread UI). + pub room_timeline: HashMap>, + /// Invite token → active invite (absent when fully consumed or never minted). + pub invites: HashMap, } impl ReducerState { @@ -225,6 +264,20 @@ impl ReducerState { .unwrap_or(false) } + /// Invite link is present, not expired, and has uses left. + pub fn invite_token_active(&self, token: &str, now_ms: i64) -> Option<&ActiveInviteState> { + let inv = self.invites.get(token)?; + if inv.uses_remaining == 0 { + return None; + } + if let Some(exp) = inv.expires_ts_ms { + if now_ms > exp { + return None; + } + } + Some(inv) + } + pub fn content_for_scope_mut(&mut self, scope: ScopeId) -> &mut ContentState { self.content.entry(scope).or_default() } @@ -356,6 +409,17 @@ impl ReducerState { visibility: rc.visibility, }, ); + self.room_timeline + .entry(rc.room_id.clone()) + .or_default() + .push(RoomTimelineEntry { + ts: rc.ts, + kind: RoomTimelineKind::RoomCreated { + owner: rc.owner.clone(), + slug: rc.slug.clone(), + visibility: rc.visibility, + }, + }); } Event::Ingest(mut ing) => { ing.thread_tag = canonicalize_tag(&ing.thread_tag); @@ -525,18 +589,31 @@ impl ReducerState { nav!(self.actor_last_post_ts, keypath(ing.principal.clone()), setval(ing.ts)); } Event::GrantAdded(ga) => { + let room_id = ga.room_id.clone(); let caps = self.grants .entry(ga.room_id) .or_default() - .entry(ga.username) + .entry(ga.username.clone()) .or_default(); - for cap in ga.capabilities { + for cap in ga.capabilities.iter().copied() { caps.insert(cap); } + self.room_timeline + .entry(room_id) + .or_default() + .push(RoomTimelineEntry { + ts: ga.ts, + kind: RoomTimelineKind::GrantAdded { + username: ga.username.clone(), + granted_by: ga.granted_by.clone(), + capabilities: ga.capabilities.clone(), + }, + }); } Event::GrantRevoked(gr) => { + let room_id = gr.room_id.clone(); if let Some(room_grants) = self.grants.get_mut(&gr.room_id) { - let username = gr.username; + let username = gr.username.clone(); if let Some(caps) = room_grants.get_mut(&username) { for cap in &gr.capabilities { caps.remove(cap); @@ -549,6 +626,37 @@ impl ReducerState { self.grants.remove(&gr.room_id); } } + self.room_timeline + .entry(room_id) + .or_default() + .push(RoomTimelineEntry { + ts: gr.ts, + kind: RoomTimelineKind::GrantRevoked { + username: gr.username.clone(), + revoked_by: gr.revoked_by.clone(), + capabilities: gr.capabilities.clone(), + }, + }); + } + Event::InviteMinted(im) => { + self.invites.insert( + im.token.clone(), + ActiveInviteState { + room_id: im.room_id.clone(), + capabilities: im.capabilities.iter().copied().collect(), + inviter: im.inviter.clone(), + uses_remaining: im.max_uses, + expires_ts_ms: im.expires_ts_ms, + }, + ); + } + Event::InviteRedeemed(ir) => { + if let Some(inv) = self.invites.get_mut(&ir.token) { + inv.uses_remaining = inv.uses_remaining.saturating_sub(1); + if inv.uses_remaining == 0 { + self.invites.remove(&ir.token); + } + } } } } @@ -570,6 +678,8 @@ impl Default for ReducerState { actor_last_post_ts: HashMap::new(), ingests_ordered: Vec::new(), grants: HashMap::new(), + room_timeline: HashMap::new(), + invites: HashMap::new(), } } } diff --git a/server/src/state.rs b/server/src/state.rs index b1ff2330903780cbe0cdf35964cb166eb2423d7d..628fd5921b34ba53ac0ca6b6fd33957ca09129b9 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -1,8 +1,20 @@ +use std::collections::HashMap; use std::sync::Arc; use tokio::sync::{broadcast, RwLock}; -use crate::{event_log::EventLog, reducer::ReducerState}; +use crate::{event_log::EventLog, events::ThreadCapability, reducer::ReducerState}; + +/// Ephemeral invite link (24h TTL, in-memory only; not written to the event log). +#[derive(Debug, Clone)] +pub struct InviteState { + pub room_id: String, + pub capabilities: Vec, + pub expires_at_ms: i64, + pub max_uses: usize, + pub current_uses: usize, + pub inviter: String, +} #[derive(Debug, Clone)] pub struct PendingSession { @@ -10,6 +22,8 @@ pub struct PendingSession { pub created_ts: i64, pub provider: Option, pub provider_id: Option, + /// When set, successful OAuth completion redeems this invite token and appends [`crate::events::GrantAdded`]. + pub redeem_invite: Option, pub complete: Option<(String /*username*/, String /*bearer*/ )>, } @@ -42,7 +56,9 @@ pub struct AppState { pub cfg: Arc, pub event_log: Arc, pub reduced: Arc>, - pub pending_sessions: Arc>>, + pub pending_sessions: Arc>>, + /// Ephemeral invite tokens (`inv_…`) until expiry or exhaustion. + pub invites: Arc>>, /// Broadcast channel for SSE live-streaming. Capacity = 64 events. pub stream_tx: broadcast::Sender, /// Broadcast channel for web SSE HTML fragments (poem pattern). Capacity = 64. @@ -58,7 +74,8 @@ impl AppState { cfg: Arc::new(cfg), event_log: Arc::new(event_log), reduced: Arc::new(RwLock::new(ReducerState::default())), - pending_sessions: Arc::new(RwLock::new(std::collections::HashMap::new())), + pending_sessions: Arc::new(RwLock::new(HashMap::new())), + invites: Arc::new(RwLock::new(HashMap::new())), stream_tx, html_tx, } diff --git a/server/src/timeline.rs b/server/src/timeline.rs new file mode 100644 index 0000000000000000000000000000000000000000..251158943ab36d3015268e35f6bdd01c3f42ab3b --- /dev/null +++ b/server/src/timeline.rs @@ -0,0 +1,153 @@ +//! Room admin lines merged into forum thread views. + +use crate::{ + canonical_path::canonicalize_tag, + reducer::{ReducerState, RoomTimelineEntry, RoomTimelineKind}, +}; + +fn cap_label(c: crate::events::ThreadCapability) -> &'static str { + use crate::events::ThreadCapability::*; + match c { + View => "view", + Post => "post", + Vote => "vote", + AddItem => "add_item", + Manage => "manage", + } +} + +fn caps_list(caps: &[crate::events::ThreadCapability]) -> String { + let mut v: Vec<_> = caps.iter().map(|c| cap_label(*c)).collect(); + v.sort(); + v.join(", ") +} + +/// Human-readable system line for the thread feed. +pub fn format_room_timeline_entry(e: &RoomTimelineEntry) -> String { + match &e.kind { + RoomTimelineKind::RoomCreated { + owner, + slug, + visibility, + } => { + let vis = match visibility { + crate::events::ThreadVisibility::Public => "public", + crate::events::ThreadVisibility::Private => "private", + }; + format!("@{owner} created room #{slug} ({vis})") + } + RoomTimelineKind::GrantAdded { + username, + granted_by, + capabilities, + } => { + format!( + "@{granted_by} granted @{} {}", + username, + caps_list(capabilities) + ) + } + RoomTimelineKind::GrantRevoked { + username, + revoked_by, + capabilities, + } => { + format!( + "@{revoked_by} revoked @{} {}", + username, + caps_list(capabilities) + ) + } + } +} + +#[derive(Clone, Debug)] +pub enum MergedThreadRow { + System { ts: i64, text: String }, + Post { + index: usize, + id: String, + ts: i64, + principal: String, + raw: String, + }, +} + +/// Merge room admin lines with thread ingests for one room + tag. Oldest first. +/// `actor_prefix` filters posts only (system lines always included). +pub fn merge_thread_rows( + reduced: &ReducerState, + room_wire: &str, + thread_tag: &str, + since: Option, + before: Option, + actor_prefix: &str, +) -> Vec { + let scope = crate::reducer::scope_from_room_wire(room_wire); + let tag = canonicalize_tag(thread_tag); + let key = (scope.clone(), tag.clone()); + + let mut rows: Vec = Vec::new(); + + if let Some(entries) = reduced.room_timeline.get(room_wire.trim()) { + for e in entries { + if since.map_or(true, |s| e.ts >= s) && before.map_or(true, |b| e.ts < b) { + rows.push(MergedThreadRow::System { + ts: e.ts, + text: format_room_timeline_entry(e), + }); + } + } + } + + let all_ids: Vec = reduced + .ingests_by_scope_thread + .get(&key) + .map(|q| q.iter().rev().cloned().collect()) + .unwrap_or_default(); + + for (idx, id) in all_ids.into_iter().enumerate() { + let Some(ing) = reduced.ingests_by_id.get(&id) else { + continue; + }; + if since.map_or(true, |s| ing.ts >= s) && before.map_or(true, |b| ing.ts < b) { + if !actor_prefix.is_empty() + && !ing + .principal + .to_lowercase() + .starts_with(actor_prefix) + { + continue; + } + rows.push(MergedThreadRow::Post { + index: idx, + id: ing.id.clone(), + ts: ing.ts, + principal: ing.principal.clone(), + raw: ing.raw.clone(), + }); + } + } + + rows.sort_by(|a, b| { + let ta = match a { + MergedThreadRow::System { ts, .. } | MergedThreadRow::Post { ts, .. } => *ts, + }; + let tb = match b { + MergedThreadRow::System { ts, .. } | MergedThreadRow::Post { ts, .. } => *ts, + }; + ta.cmp(&tb) + }); + rows +} + +/// Public forum thread (`room_wire == "public"`): same merge (timeline usually empty). +pub fn merge_public_thread_rows( + reduced: &ReducerState, + thread_tag: &str, + since: Option, + before: Option, + actor_prefix: &str, +) -> Vec { + merge_thread_rows(reduced, "public", thread_tag, since, before, actor_prefix) +} diff --git a/test/grants.bb b/test/grants.bb index 10ef0f08ce1c3945e5f1634314e6aebcf6810cd2..4ded5ce92576cdea653cce0ba8a751be4374731a 100644 --- a/test/grants.bb +++ b/test/grants.bb @@ -105,7 +105,7 @@ ;; Alice grants bob View only. (println "\nalice grants bob View only…") (assert! (rpc-line-ok? (:parsed (rpc-batch! base-url alice-token - [{"RoomGrant" {"room" room-id "username" "bob" "capability" "view"}}]))) + [{"RoomGrant" {"room" room-id "username" "bob" "capabilities" ["view"]}}]))) "grant View RPC ok") (println "\nbob (View only) tries to post prose…") @@ -117,7 +117,7 @@ ;; Alice grants bob Post. (println "\nalice grants bob Post…") (assert! (rpc-line-ok? (:parsed (rpc-batch! base-url alice-token - [{"RoomGrant" {"room" room-id "username" "bob" "capability" "post"}}]))) + [{"RoomGrant" {"room" room-id "username" "bob" "capabilities" ["post"]}}]))) "grant Post RPC ok") (println "\nbob (View + Post) posts prose…") @@ -143,7 +143,7 @@ ;; Alice grants bob Vote. (println "\nalice grants bob Vote…") (assert! (rpc-line-ok? (:parsed (rpc-batch! base-url alice-token - [{"RoomGrant" {"room" room-id "username" "bob" "capability" "vote"}}]))) + [{"RoomGrant" {"room" room-id "username" "bob" "capabilities" ["vote"]}}]))) "grant Vote RPC ok") (println "\nbob (View + Post + Vote) votes…") diff --git a/test/invites.bb b/test/invites.bb new file mode 100644 index 0000000000000000000000000000000000000000..f72b1e630a6856174cdef11eb977e2f81b16e551 --- /dev/null +++ b/test/invites.bb @@ -0,0 +1,150 @@ +(ns test.invites + "Ephemeral invite links: mint via RPC, GET /join → pending session + OAuth, redemption → GrantAdded, + RoomAudit, post succeeds, second GET /join returns 404 when max_uses exhausted." + (:require [babashka.fs :as fs] + [cheshire.core :as json] + [clojure.string :as str] + [test.common :as common] + [test.oauth :as oauth])) + +(def ^:private counts (atom {:pass 0 :fail 0})) + +(defn- assert! [pred msg] + (common/test-assert! counts pred msg)) + +(defn- bearer [token] {"Authorization" (str "Bearer " token)}) + +(defn- rpc-batch! [base-url token cmds] + (let [resp (oauth/http-post-json (str base-url "/api/v0/rpc") cmds :headers (bearer token))] + {:status (:status resp) + :parsed (json/parse-string (:body resp) false)})) + +(defn- rpc-line-ok? [parsed] + (true? (get-in parsed ["results" 0 "ok"]))) + +(defn- session-from-login-location [loc] + (when loc + (let [qpart (if (str/includes? loc "?") + (-> loc (str/split #"\?" 2) second) + "") + qpart (if (str/includes? qpart "#") + (-> qpart (str/split #"#" 2) first) + qpart) + m (oauth/parse-query qpart)] + (some-> (get m :session) str)))) + +(defn- invite-token-from-url [invite-url] + (second (re-find #"/join/(inv_[^/?#]+)" (str invite-url)))) + +(defn- register-user! [base-url session-agent username] + (oauth/complete-registration! base-url + :agent session-agent + :username username + :assert! (fn [pred msg] (assert! pred msg)))) + +(defn- ingest! [base-url token room thread delegate text] + (rpc-batch! base-url token + [{"Post" {"room" room + "thread_tag" thread + "delegate" delegate + "text" text + "return_rank_diff" false}}])) + +(defn invites-test [& _args] + (println "\n━━━ ephemeral invite + audit integration check ━━━\n") + (reset! counts {:pass 0 :fail 0}) + + (println "building server binary…") + (common/letlocals + (bind build (common/run-cargo-build-release! ["slugsocial-server"])) + (assert! (zero? (:exit build)) "cargo build succeeds") + (bind server-bin "target/release/slugsocial-server") + + (bind tmp-dir (str (fs/create-temp-dir {:prefix "slug-invites-"}))) + (bind slug-port (common/pick-port)) + (bind google-port (common/pick-port)) + (bind base-url (str "http://127.0.0.1:" slug-port)) + (bind google-url (str "http://127.0.0.1:" google-port)) + + (bind !server (atom nil)) + (bind !google (atom nil)) + + (bind server-env (common/slug-server-env tmp-dir base-url google-url slug-port)) + (try + (println (str "starting mock google on :" google-port)) + (reset! !google (oauth/start-mock-google google-port + :google-users ["google-user-alice" "google-user-bob"])) + + (println (str "starting server on :" slug-port)) + (reset! !server (common/start-server server-bin server-env)) + (assert! (common/wait-for-server base-url 10000) "server responds to /healthz") + + (println "\nregistering alice…") + (let [alice-token (register-user! base-url + "00000000-0000-0000-0000-000000000001:test:local/dev" + "alice") + + _ (println "\nalice creates private room…") + create (rpc-batch! base-url alice-token + [{"RoomCreate" {"slug" "invite-demo" "visibility" "private"}}]) + _ (assert! (= 200 (:status create)) "room create HTTP 200") + _ (assert! (rpc-line-ok? (:parsed create)) "room create RPC ok") + room-id (get-in (:parsed create) ["results" 0 "result" "RoomCreated" "room_id"]) + _ (assert! (some? room-id) "room_id present") + + _ (println "\nalice mints invite (view,post,vote uses=1)…") + mint (rpc-batch! base-url alice-token + [{"RoomMintInvite" {"room" room-id + "capabilities" ["view" "post" "vote"] + "max_uses" 1}}]) + _ (assert! (= 200 (:status mint)) "mint HTTP 200") + _ (assert! (rpc-line-ok? (:parsed mint)) "mint RPC ok") + invite-url (get-in (:parsed mint) ["results" 0 "result" "RoomInviteMinted" "invite_url"]) + inv-tok (invite-token-from-url invite-url) + _ (assert! (some? inv-tok) "invite token parsed from URL") + + _ (println "\nGET /join/:token (expect redirect + session)…") + join-resp (oauth/http-get-no-redirect (str base-url "/join/" inv-tok)) + _ (assert! (contains? #{302 307} (:status join-resp)) + (str "join returns redirect (got status " (:status join-resp) ")")) + sess (session-from-login-location (:location join-resp)) + _ (assert! (and (some? sess) (str/starts-with? sess "p_")) "Location carries session=p_…") + + _ (println "\nbob completes OAuth via invite session…") + bob-token (oauth/complete-pending-session! base-url sess "bob" + :assert! (fn [pred msg] (assert! pred msg))) + + _ (println "\nalice runs RoomAudit…") + audit (rpc-batch! base-url alice-token [{"RoomAudit" {"room" room-id}}]) + _ (assert! (rpc-line-ok? (:parsed audit)) "audit RPC ok") + grants (get-in (:parsed audit) ["results" 0 "result" "RoomAudit" "grants"]) + bob-entry (first (filter #(= "bob" (get % "username")) grants)) + _ (assert! (some? bob-entry) "audit lists bob") + bob-caps (set (get bob-entry "capabilities")) + _ (assert! (= bob-caps #{"view" "post" "vote"}) "bob has view, post, vote") + + _ (println "\nbob posts prose to private room…") + _ (assert! (rpc-line-ok? (:parsed (ingest! base-url bob-token room-id "main" + "00000000-0000-0000-0000-000000000002:test:local/dev" + "Hello via invite link."))) + "bob post succeeds") + + _ (println "\nsecond GET /join (invite exhausted → 404)…") + join2 (oauth/http-get-no-redirect (str base-url "/join/" inv-tok)) + _ (assert! (= 404 (:status join2)) "exhausted invite returns 404")] + + (println "\ninvite lifecycle OK.")) + + (finally + (when-some [s @!server] (common/kill-server s)) + (when-some [g @!google] ((:stop-fn g))) + (fs/delete-tree tmp-dir))) + + (bind {pass :pass fail :fail} @counts) + (if (zero? fail) + (println (str "\n" common/ansi-green "━━━ " pass " invite checks passed ━━━" common/ansi-reset "\n")) + (do (println (str "\n" common/ansi-red "━━━ " fail " invite checks FAILED ━━━" common/ansi-reset "\n")) + (System/exit 1))))) + +(when (= *file* (System/getProperty "babashka.file")) + (invites-test)) diff --git a/test/oauth.bb b/test/oauth.bb index b69459acd17c23c023170d8dd93fe45bb49c8d88..efb3f3e66346ffde81bf9622d75c3a6ec89f8db2 100644 --- a/test/oauth.bb +++ b/test/oauth.bb @@ -21,6 +21,21 @@ resp (.send (http-client) req (java.net.http.HttpResponse$BodyHandlers/ofString))] {:status (.statusCode resp) :body (.body resp) :headers (.map (.headers resp))}))) +(defn http-get-no-redirect + "GET without following redirects; returns `:location` from the first `Location` header when present." + [url & {:keys [headers]}] + (let [client (-> (java.net.http.HttpClient/newBuilder) + (.followRedirects java.net.http.HttpClient$Redirect/NEVER) + (.connectTimeout connect-timeout) + (.build)) + b (java.net.http.HttpRequest/newBuilder (java.net.URI/create url))] + (doseq [[k v] (or headers {})] + (.header b k v)) + (let [req (-> b (.timeout request-timeout) (.GET) (.build)) + resp (.send client req (java.net.http.HttpResponse$BodyHandlers/ofString)) + loc (first (get (.map (.headers resp)) "location"))] + {:status (.statusCode resp) :body (.body resp) :location loc}))) + (defn http-post-json [url data & {:keys [headers]}] (let [body (json/generate-string data) b (java.net.http.HttpRequest/newBuilder (java.net.URI/create url))] @@ -50,6 +65,35 @@ resp (.send (http-client) req (java.net.http.HttpResponse$BodyHandlers/ofString))] {:status (.statusCode resp) :body (.body resp) :headers (.map (.headers resp))}))) +(defn complete-pending-session! + "Finish OAuth for an existing pending session id (e.g. created by `GET /join/inv_…`). Returns bearer token." + [base-url session-id username & {:keys [assert!]}] + (let [check! (fn [pred msg resp] + (if assert! + (assert! pred msg) + (when-not pred + (throw (ex-info msg {:resp resp})))))] + (let [enc (java.net.URLEncoder/encode session-id "UTF-8") + login-url (str base-url "/auth/login?session=" enc) + login-get (http-get login-url)] + (check! (= 200 (:status login-get)) + (str "oauth redirect chain for session " session-id) + login-get) + (let [choose (http-post-form (str base-url "/auth/choose-username") + {:session session-id :username username})] + (check! (= 200 (:status choose)) + (str "choose-username for " username " returns 200") + choose) + (let [poll (http-get (str base-url "/api/v0/pending-session/" session-id))] + (check! (= 200 (:status poll)) + (str "pending-session poll returns 200") + poll) + (let [poll-json (json/parse-string (:body poll) true)] + (check! (:complete poll-json) + (str "pending session complete for " username) + poll) + (:token poll-json))))))) + (defn parse-query [s] (into {} (for [part (str/split (or s "") #"&") diff --git a/types/src/lib.rs b/types/src/lib.rs index a715fe5639f4a3a7bfedcc706b5fb312b98c9ac5..98c66a00fd27803ef6f75b5ac478ff2eb762d771 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -123,13 +123,32 @@ pub struct PathDetailResponse { #[derive(Debug, Serialize, Deserialize)] pub struct ThreadDetailResponse { pub thread: String, - pub posts: Vec, - /// Total posts in this thread. + /// Chronological page: prose posts and room system lines, oldest first within the window. + pub items: Vec, + /// Total rows (posts + system lines) in this thread after filters. pub total: usize, - /// Chronological offset of the first post in this page. + /// Offset into the merged chronological list. pub offset: usize, } +/// One row in a thread timeline: a normal post or a room system line. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ThreadItem { + Post { + id: String, + index: usize, + ts: i64, + actor: String, + body: String, + truncated: bool, + }, + System { + ts: i64, + text: String, + }, +} + /// One post in a thread. Full body, no snippet. #[derive(Debug, Serialize, Deserialize)] pub struct PostRow { @@ -224,6 +243,23 @@ pub struct FeedPost { // RPC batch API (`POST /api/v0/rpc`) // --------------------------------------------------------------------------- +fn default_invite_max_uses() -> usize { + 1 +} + +/// One principal's capabilities in a private room (from [`RpcCommand::RoomAudit`]). +#[derive(Debug, Serialize, Deserialize)] +pub struct RoomAuditEntry { + pub username: String, + pub capabilities: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct RoomAuditResponse { + pub room: String, + pub grants: Vec, +} + #[derive(Debug, Serialize, Deserialize)] #[serde(transparent)] pub struct RpcBatch(pub Vec); @@ -287,10 +323,27 @@ pub enum RpcCommand { visibility: Option, }, RoomGrant { + room: String, + username: String, + /// Capability names: `view`, `post`, `vote`, `add_item`, `manage`. + capabilities: Vec, + }, + RoomRevoke { room: String, username: String, capability: String, }, + /// Mint a shareable invite link (24h TTL, stored in memory only until redeemed or expiry). + RoomMintInvite { + room: String, + capabilities: Vec, + #[serde(default = "default_invite_max_uses")] + max_uses: usize, + }, + /// List principals granted access in a room (requires View or Manage). + RoomAudit { + room: String, + }, GetGlobalRank { room: String, #[serde(default)] @@ -361,6 +414,13 @@ pub enum RpcResult { RoomCreated { room_id: String, }, + RoomInviteMinted { + invite_url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + expires_at_ms: Option, + max_uses: usize, + }, + RoomAudit(RoomAuditResponse), GrantOk {}, GlobalRank(GlobalRankResponse), Pair(PairResponse), Side B — contributor: tommy-mor Side B — commit message: [1d14ff09] Ephemeral Reddit content; log structure only (#46) * Keep Reddit content ephemeral; log structure only Remove EntityImported and EntityStore. Reddit fetches write display content directly to the projection with a fetched_at timestamp, while the event log records NodeEnsured for discovered identities only. A background task evicts cached display content after 48 hours. Votes, tree structure, and ItemIds remain in the log and projection. Co-authored-by: tommy * Fix reddit import test assertions and Clojure syntax Co-authored-by: tommy --------- Co-authored-by: Cursor Agent Side B — unified diff (full patch): diff --git a/server/src/bin/storage_bench.rs b/server/src/bin/storage_bench.rs index 4762d0d23bc8e2df439afe09a3235ba7f72bd486..3d988416ad36b27a7d3dc84280cfdbdcafa43e69 100644 --- a/server/src/bin/storage_bench.rs +++ b/server/src/bin/storage_bench.rs @@ -6,8 +6,8 @@ use std::{ }; use sorter2_server::{ - entity_store::EntityStore, event_log::EventLog, events::Event, journal::JournalClient, - projection_apply, projection_store::ProjectionStore, + event_log::EventLog, events::Event, journal::JournalClient, projection_apply, + projection_store::ProjectionStore, }; #[tokio::main] @@ -18,12 +18,10 @@ async fn main() -> Result<(), Box> { let data_dir = opts.data_dir.to_string_lossy().into_owned(); let event_log = Arc::new(EventLog::new(format!("{data_dir}/events.jsonl"))); let db = durable::Db::open(opts.data_dir.join("store"))?; - let entity_store = EntityStore::from_db(&db)?; let projection_store = ProjectionStore::from_db(&db)?; let journal = JournalClient::spawn( event_log.clone(), - entity_store.clone(), projection_store.clone(), event_log.last_sequence().await? + 1, ); @@ -46,12 +44,9 @@ async fn main() -> Result<(), Box> { drop(journal); let rebuild_start = Instant::now(); - entity_store.reset()?; projection_store.reset()?; let rebuild = event_log - .replay(|record| { - projection_apply::apply_records(&projection_store, &entity_store, &[record]) - }) + .replay(|record| projection_apply::apply_records(&projection_store, &[record])) .await?; let rebuild_elapsed = rebuild_start.elapsed(); diff --git a/server/src/entity_store.rs b/server/src/entity_store.rs deleted file mode 100644 index d29f39aecca6f12cdcf263cf77c3654eb4ee6cfa..0000000000000000000000000000000000000000 --- a/server/src/entity_store.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! Off-heap storage for full entity payloads (Reddit API JSON). -//! -//! Derived [`crate::reducer::EntityData`] is stored on the node; the raw JSON -//! lives here, in the shared durable [`Store`] schema. - -use std::path::Path; - -use durable::{Batch, Db, Durability}; -use serde_json::Value; - -use crate::{ - path_types::ItemId, - storage_dto::{decode_entity_payload, encode_entity_payload}, - storage_schema::{Store, StoreFields}, -}; - -const ENTITY_SCHEMA_KEY: &str = "schema_version"; -const ENTITY_SCHEMA_VERSION: u64 = 2; - -#[derive(Debug, thiserror::Error)] -pub enum EntityStoreError { - #[error("durable error: {0}")] - Durable(#[from] durable::Error), - #[error("json error: {0}")] - Json(#[from] serde_json::Error), - #[error("storage decode error: {0}")] - Storage(String), - #[error("io error: {0}")] - Io(#[from] std::io::Error), -} - -/// Disk-backed map of entity id → raw JSON payload. -#[derive(Clone)] -pub struct EntityStore { - db: Db, -} - -impl EntityStore { - /// Open (or create) the entity database under `dir`. - pub fn open(dir: &Path) -> Result { - std::fs::create_dir_all(dir)?; - let db = Db::open(dir)?; - Self::from_db(&db) - } - - /// Create an entity store backed by an already-open database. - pub fn from_db(db: &Db) -> Result { - let store = Self { db: db.clone() }; - let version = Store::root() - .entity_meta() - .key(&ENTITY_SCHEMA_KEY.to_string()) - .get(db)?; - if version != Some(ENTITY_SCHEMA_VERSION) { - store.reset()?; - } - Ok(store) - } - - /// Clear rebuildable entity payloads and reset storage schema metadata. - pub fn reset(&self) -> Result<(), EntityStoreError> { - let root = Store::root(); - self.db.apply( - &[root.entities().clear(), root.entity_meta().clear()], - Durability::SyncWal, - )?; - self.db.run( - root.entity_meta() - .key(&ENTITY_SCHEMA_KEY.to_string()) - .set(&ENTITY_SCHEMA_VERSION), - Durability::SyncWal, - )?; - Ok(()) - } - - /// Persist a payload for `id` (overwrites any existing entry). - pub fn put(&self, id: &ItemId, payload: &Value) -> Result<(), EntityStoreError> { - self.db.run( - Store::root() - .entities() - .key(&id.as_str().to_string()) - .set(&encode_entity_payload(payload)), - Durability::SyncWal, - )?; - Ok(()) - } - - /// Add a payload write to the caller's batch. - pub fn put_in_batch( - &self, - batch: &mut Batch, - id: &ItemId, - payload: &Value, - ) -> Result<(), EntityStoreError> { - batch.write( - Store::root() - .entities() - .key(&id.as_str().to_string()) - .set(&encode_entity_payload(payload)), - ); - Ok(()) - } - - /// Load a stored payload, if present. - pub fn get(&self, id: &ItemId) -> Result, EntityStoreError> { - match Store::root() - .entities() - .key(&id.as_str().to_string()) - .get(&self.db)? - { - Some(record) => decode_entity_payload(record) - .map(Some) - .map_err(EntityStoreError::Storage), - None => Ok(None), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn round_trip_payload() { - let tmp = tempfile::tempdir().unwrap(); - let store = EntityStore::open(tmp.path()).unwrap(); - let id = ItemId::from_url("https://reddit.com/r/rust").unwrap(); - let payload = json!({"kind": "t5", "data": {"display_name": "rust"}}); - - store.put(&id, &payload).unwrap(); - let loaded = store.get(&id).unwrap().unwrap(); - assert_eq!(loaded, payload); - } -} diff --git a/server/src/events.rs b/server/src/events.rs index a3d88e285c645a96d33e6f0aed1b487b843aceb8..d76c3bb4277216b0d39c9422ba7a50db10a95e05 100644 --- a/server/src/events.rs +++ b/server/src/events.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; -use serde_json::Value; /// Schema version for JSONL log records. Bump when event semantics change. pub const CURRENT_LOG_SCHEMA: u32 = 1; @@ -30,7 +29,7 @@ pub type ViewRecord = LogRecord; /// Wall-clock timestamp carried on the log envelope for domain events. pub fn event_timestamp(event: &Event) -> i64 { match event { - Event::VoteRecorded { ts, .. } | Event::EntityImported { ts, .. } => *ts, + Event::VoteRecorded { ts, .. } => *ts, Event::NodeEnsured { .. } => crate::fetch::now_ms(), } } @@ -57,6 +56,4 @@ pub enum Event { }, /// Register a node path in the fractal tree (no external fetch). NodeEnsured { id: String }, - /// Full upstream API payload for a node (domain-specific view derived at replay/render time). - EntityImported { id: String, ts: i64, payload: Value }, } diff --git a/server/src/journal.rs b/server/src/journal.rs index 50bc89f976edb82b7b0e49e954a8eccbbe82bf87..d50023aca7c3e74068de988449b11baee32d1ecf 100644 --- a/server/src/journal.rs +++ b/server/src/journal.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use tokio::sync::{mpsc, oneshot}; use crate::{ - entity_store::EntityStore, event_log::EventLog, events::{event_timestamp, Event, EventRecord}, projection_apply, @@ -25,7 +24,6 @@ pub struct JournalClient { impl JournalClient { pub fn spawn( event_log: Arc, - entity_store: EntityStore, projection_store: ProjectionStore, next_seq: u64, ) -> Self { @@ -33,7 +31,6 @@ impl JournalClient { tokio::spawn(journal_worker( rx, event_log, - entity_store, projection_store, next_seq, )); @@ -62,7 +59,6 @@ impl JournalClient { async fn journal_worker( mut rx: mpsc::Receiver, event_log: Arc, - entity_store: EntityStore, projection_store: ProjectionStore, mut next_seq: u64, ) { @@ -75,7 +71,6 @@ async fn journal_worker( let result = append_and_project_batch( &event_log, &projection_store, - &entity_store, &mut next_seq, &batch, ) @@ -99,7 +94,6 @@ async fn journal_worker( async fn append_and_project_batch( event_log: &EventLog, projection_store: &ProjectionStore, - entity_store: &EntityStore, next_seq: &mut u64, commands: &[JournalCommand], ) -> Result<(), String> { @@ -117,7 +111,7 @@ async fn append_and_project_batch( .await .map_err(|e| e.to_string())?; *next_seq = seq; - projection_apply::apply_records(projection_store, entity_store, &records) + projection_apply::apply_records(projection_store, &records) .map_err(|e| format!("projection apply failed after durable append: {e}")) } @@ -132,10 +126,9 @@ mod tests { let log_path = tmp.path().join("events.jsonl"); let event_log = Arc::new(EventLog::new(log_path)); let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - let journal = JournalClient::spawn(event_log, entity_store, projection_store.clone(), 1); + let journal = JournalClient::spawn(event_log, projection_store.clone(), 1); let j1 = journal.clone(); let j2 = journal.clone(); @@ -177,11 +170,9 @@ mod tests { .unwrap(); let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); projection_apply::apply_records( &projection_store, - &entity_store, &[EventRecord::new( 1, 1, @@ -196,7 +187,6 @@ mod tests { let journal = JournalClient::spawn( event_log.clone(), - entity_store, projection_store.clone(), next_seq, ); @@ -219,10 +209,9 @@ mod tests { let log_path = tmp.path().join("events.jsonl"); let event_log = Arc::new(EventLog::new(log_path)); let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); let journal = - JournalClient::spawn(event_log.clone(), entity_store, projection_store.clone(), 1); + JournalClient::spawn(event_log.clone(), projection_store.clone(), 1); journal .append_many(vec![ diff --git a/server/src/lib.rs b/server/src/lib.rs index 5c02c8e704e4664453bad75d819df8a067668176..3dfc7c8acb8ed61bb73ade63e72768e402042cc5 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,5 +1,4 @@ pub mod api; -pub mod entity_store; pub mod event_log; pub mod events; pub mod fetch; diff --git a/server/src/projection_apply.rs b/server/src/projection_apply.rs index 5644557a41b3e9497c7421b444155ae629fa79f1..9c8990a8af927f35d3344c8d0872a516aba56b86 100644 --- a/server/src/projection_apply.rs +++ b/server/src/projection_apply.rs @@ -1,22 +1,20 @@ //! Apply event-log records to the durable projection as precise point updates. //! //! Each batch of records lowers to reified durable writes (edge merges, child -//! links, voted-pair flags, recent-vote pushes, entity payloads) plus a cursor -//! advance, all committed in one atomic `DisableWal` batch. The cursor moving in -//! the same batch as the (non-idempotent) edge merges guarantees exactly-once -//! application across replay. +//! links, voted-pair flags, recent-vote pushes) plus a cursor advance, all +//! committed in one atomic `DisableWal` batch. The cursor moving in the same +//! batch as the (non-idempotent) edge merges guarantees exactly-once application +//! across replay. use std::collections::BTreeSet; use crate::{ - entity_store::EntityStore, event_log::EventLogError, events::{Event, EventRecord}, path_types::ItemId, projection_store::ProjectionStore, - reddit::entity_view_from_payload, reducer::VoteData, - storage_schema::{ensure_path_writes, entity_view_writes, vote_writes}, + storage_schema::{ensure_path_writes, vote_writes}, }; fn parse_event_id(id: &str) -> Result { @@ -38,7 +36,6 @@ fn parent_from_event_scope(scope: &str) -> ItemId { pub fn apply_records( projection_store: &ProjectionStore, - entity_store: &EntityStore, records: &[EventRecord], ) -> Result<(), EventLogError> { if records.is_empty() { @@ -79,14 +76,6 @@ pub fn apply_records( let parsed = parse_event_id(id)?; ensure_path_writes(&mut batch, &parsed); } - Event::EntityImported { id, payload, .. } => { - let parsed = parse_event_id(id)?; - let view = entity_view_from_payload(&parsed, payload); - entity_view_writes(&mut batch, &parsed, view.as_ref()); - entity_store - .put_in_batch(&mut batch, &parsed, payload) - .map_err(|e| EventLogError::Apply(e.to_string()))?; - } } last_seq = record.seq; } diff --git a/server/src/projection_store.rs b/server/src/projection_store.rs index 30ee478f953e80d7322bdbfa521ae559ff08fa51..8576d671f351004426207894ac35594ddb0f70cf 100644 --- a/server/src/projection_store.rs +++ b/server/src/projection_store.rs @@ -9,13 +9,16 @@ use durable::{Db, Durability, Write}; use crate::{ path_types::ItemId, - reducer::{GlobalTree, NodeState}, - storage_schema::{load_node_state, node, NodeSchemaFields, Store, StoreFields}, + reducer::{EntityData, GlobalTree, NodeState}, + storage_schema::{ + entity_content_clear_writes, entity_content_writes, load_node_state, node, NodeSchemaFields, + Store, StoreFields, + }, }; const PROJECTION_CURSOR_KEY: &str = "cursor"; const PROJECTION_SCHEMA_KEY: &str = "schema_version"; -const PROJECTION_SCHEMA_VERSION: u64 = 2; +const PROJECTION_SCHEMA_VERSION: u64 = 3; #[derive(Debug, thiserror::Error)] pub enum ProjectionStoreError { @@ -148,6 +151,45 @@ impl ProjectionStore { )?; Ok(()) } + + /// Cache Reddit display content outside the event log (must be evicted per policy). + pub fn put_ephemeral_content( + &self, + id: &ItemId, + view: &EntityData, + fetched_at: i64, + ) -> Result<(), ProjectionStoreError> { + let mut batch = self.db.batch(); + entity_content_writes(&mut batch, id, view, fetched_at); + batch + .commit_with(Durability::DisableWal) + .map_err(ProjectionStoreError::from)?; + Ok(()) + } + + /// Drop cached display content older than `cutoff_ms` (votes and tree structure remain). + pub fn evict_content_older_than(&self, cutoff_ms: i64) -> Result { + let keys = Store::root().nodes().keys(&self.db)?; + let mut batch = self.db.batch(); + let mut evicted = 0usize; + for key in keys { + let id = parse_node_key(&key)?; + let np = node(&id); + let Some(fetched_at) = np.fetched_at().get(&self.db)? else { + continue; + }; + if fetched_at > 0 && fetched_at < cutoff_ms { + entity_content_clear_writes(&mut batch, &id); + evicted += 1; + } + } + if evicted > 0 { + batch + .commit_with(Durability::DisableWal) + .map_err(ProjectionStoreError::from)?; + } + Ok(evicted) + } } fn parse_node_key(key: &str) -> Result { @@ -162,7 +204,7 @@ fn parse_node_key(key: &str) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::{entity_store::EntityStore, events::Event, projection_apply}; + use crate::{events::Event, projection_apply, reducer::EntityData}; fn record(seq: u64, event: Event) -> crate::events::EventRecord { crate::events::EventRecord::new(seq, crate::events::event_timestamp(&event), event) @@ -172,7 +214,6 @@ mod tests { fn applies_and_loads_reducer_nodes() { let tmp = tempfile::tempdir().unwrap(); let db = Db::open(tmp.path()).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let store = ProjectionStore::from_db(&db).unwrap(); let event = Event::VoteRecorded { @@ -183,7 +224,7 @@ mod tests { ratio_right: 1, scope: String::new(), }; - projection_apply::apply_records(&store, &entity_store, &[record(1, event)]).unwrap(); + projection_apply::apply_records(&store, &[record(1, event)]).unwrap(); assert_eq!(store.last_applied_event_count().unwrap(), 1); let loaded = store.load_tree().unwrap(); @@ -196,7 +237,6 @@ mod tests { fn hydrates_scope_with_child_nodes() { let tmp = tempfile::tempdir().unwrap(); let db = Db::open(tmp.path()).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let store = ProjectionStore::from_db(&db).unwrap(); let event = Event::VoteRecorded { @@ -207,11 +247,36 @@ mod tests { ratio_right: 1, scope: String::new(), }; - projection_apply::apply_records(&store, &entity_store, &[record(1, event)]).unwrap(); + projection_apply::apply_records(&store, &[record(1, event)]).unwrap(); let scoped = store.scope_tree(&ItemId::root()).unwrap(); let root = scoped.get(&ItemId::root()).unwrap(); assert_eq!(root.children.len(), 2); assert!(scoped.get(&ItemId::opaque("alpha")).is_some()); } + + #[test] + fn evicts_stale_ephemeral_content() { + let tmp = tempfile::tempdir().unwrap(); + let db = Db::open(tmp.path()).unwrap(); + let store = ProjectionStore::from_db(&db).unwrap(); + let id = ItemId::from_url("https://reddit.com/r/rust").unwrap(); + store + .put_ephemeral_content( + &id, + &EntityData { + title: "Rust".into(), + author: None, + body_html: None, + thumb_url: None, + image_url: None, + link_url: None, + }, + 1_000, + ) + .unwrap(); + assert!(store.load_node(&id).unwrap().unwrap().data.is_some()); + assert_eq!(store.evict_content_older_than(2_000).unwrap(), 1); + assert!(store.load_node(&id).unwrap().unwrap().data.is_none()); + } } diff --git a/server/src/reddit.rs b/server/src/reddit.rs index 20b7f9e3f8be39268a1767d09f5cf81eaa6ae0df..a874814f8927192ee62cab2d0db1efd27dcd57b7 100644 --- a/server/src/reddit.rs +++ b/server/src/reddit.rs @@ -9,10 +9,13 @@ use serde_json::Value; use tokio::sync::{mpsc, oneshot}; use crate::{ - entity_store::EntityStore, events::Event, fetch::now_ms, journal::JournalClient, - path_types::ItemId, reducer::GlobalTree, + events::Event, fetch::now_ms, journal::JournalClient, + path_types::ItemId, projection_store::ProjectionStore, }; +/// Reddit display content must not be retained longer than this (API policy). +pub const REDDIT_CONTENT_TTL: std::time::Duration = std::time::Duration::from_secs(48 * 3600); + #[derive(Debug, Clone, PartialEq, Eq)] pub enum FetchJobResult { /// Number of entities written (1 for self, N for children). @@ -70,7 +73,11 @@ struct OAuthToken { } impl RedditBroker { - pub fn spawn(journal: JournalClient, config: RedditApiConfig) -> Self { + pub fn spawn( + journal: JournalClient, + projection_store: ProjectionStore, + config: RedditApiConfig, + ) -> Self { let (tx, rx) = mpsc::channel(100); let mut headers = header::HeaderMap::new(); @@ -93,7 +100,7 @@ impl RedditBroker { "reddit worker started" ); - tokio::spawn(reddit_worker(rx, journal, client, config)); + tokio::spawn(reddit_worker(rx, journal, projection_store, client, config)); Self { tx } } @@ -189,27 +196,50 @@ pub fn entity_view_from_payload( None } -pub fn apply_entity_import( - tree: &mut GlobalTree, - store: &EntityStore, - id: &ItemId, - payload: Value, -) -> Result<(), String> { - let view = entity_view_from_payload(id, &payload); - store.put(id, &payload).map_err(|e| e.to_string())?; - tree.apply_entity(id, view); - Ok(()) -} - fn notify(done: Option>, result: FetchJobResult) { if let Some(tx) = done { let _ = tx.send(result); } } +async fn import_fetched_payload( + kind: FetchKind, + fetch_id: &ItemId, + payload: Value, + projection_store: &ProjectionStore, + journal: &JournalClient, +) -> Result { + let fetched_at = now_ms(); + let imports: Vec<(ItemId, Value)> = match kind { + FetchKind::SelfEntity => vec![(fetch_id.clone(), payload)], + FetchKind::Children => parse_children(fetch_id, &payload), + }; + + for (id, child_payload) in &imports { + if let Some(view) = entity_view_from_payload(id, child_payload) { + projection_store + .put_ephemeral_content(id, &view, fetched_at) + .map_err(|e| e.to_string())?; + } + } + + let events: Vec = imports + .iter() + .map(|(id, _)| Event::NodeEnsured { + id: id.as_str().to_string(), + }) + .collect(); + let written = events.len(); + if !events.is_empty() { + journal.append_many(events).await?; + } + Ok(written) +} + async fn reddit_worker( mut rx: mpsc::Receiver, journal: JournalClient, + projection_store: ProjectionStore, client: Client, config: RedditApiConfig, ) { @@ -276,33 +306,20 @@ async fn reddit_worker( match outcome { Ok(FetchOutcome::Payload(payload)) => { - let imports: Vec<(ItemId, Value)> = match kind { - FetchKind::SelfEntity => vec![(fetch_id.clone(), payload)], - FetchKind::Children => parse_children(&fetch_id, &payload), - }; tracing::debug!( item = %fetch_id, ?kind, - count = imports.len(), - "reddit fetch got payload, importing" + "reddit fetch got payload, caching ephemerally" ); - let events: Vec = imports - .into_iter() - .map(|(child_id, child_payload)| Event::EntityImported { - id: child_id.as_str().to_string(), - ts: now_ms(), - payload: child_payload, - }) - .collect(); - let written = events.len(); - - match journal.append_many(events).await { + match import_fetched_payload(kind, &fetch_id, payload, &projection_store, &journal) + .await + { Err(e) => { - tracing::warn!(item = %fetch_id, err = %e, "reddit import journal failed"); + tracing::warn!(item = %fetch_id, err = %e, "reddit import failed"); notify(done, FetchJobResult::Failed(e)); } - Ok(()) => { + Ok(written) => { recently_fetched.insert(key.clone(), Instant::now()); current_delay = Duration::from_millis(600); tracing::info!(item = %fetch_id, ?kind, written, "reddit import complete"); diff --git a/server/src/reducer.rs b/server/src/reducer.rs index 1352a8f0771add3d868a1b30109b087a2a6dba6f..0c75c85150bb9e5f578bbadf58b3e43f8a80be4b 100644 --- a/server/src/reducer.rs +++ b/server/src/reducer.rs @@ -136,8 +136,7 @@ pub struct EntityData { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct NodeState { pub id: ItemId, - /// Domain-specific view derived from imported payload (e.g. Reddit title/author). - /// Raw JSON lives in [`crate::entity_store::EntityStore`]. + /// Ephemeral display view (Reddit title/author/etc.; not event-logged). pub data: Option, pub children: HashSet, pub local_ranking: GroupState, diff --git a/server/src/state.rs b/server/src/state.rs index e44849eec46123072b238afd40a1fdd51ce19bd9..247b9047a57956f76c4b6bef691662101e62a8f9 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -1,14 +1,14 @@ use std::{error::Error, sync::Arc}; use crate::{ - entity_store::EntityStore, event_log::EventLog, events::Event, + fetch::now_ms, journal::JournalClient, path_types::ItemId, projection_apply, projection_store::ProjectionStore, - reddit::{RedditApiConfig, RedditBroker}, + reddit::{RedditApiConfig, RedditBroker, REDDIT_CONTENT_TTL}, reducer::{GlobalTree, VoteData}, view_log::ViewLog, views::ViewStore, @@ -45,7 +45,6 @@ pub fn normalize_scope(raw: &str) -> String { async fn catch_up_projection( event_log: &EventLog, - entity_store: &EntityStore, projection_store: &ProjectionStore, ) -> Result<(), crate::event_log::EventLogError> { let after_seq = projection_store @@ -54,7 +53,7 @@ async fn catch_up_projection( let stats = event_log .replay_from(after_seq, |record| { - projection_apply::apply_records(projection_store, entity_store, &[record]) + projection_apply::apply_records(projection_store, &[record]) }) .await?; if after_seq > stats.last_seq { @@ -67,22 +66,34 @@ async fn catch_up_projection( Ok(()) } +fn spawn_content_evictor(projection_store: ProjectionStore) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(15 * 60)); + interval.tick().await; + loop { + interval.tick().await; + let cutoff = now_ms() - REDDIT_CONTENT_TTL.as_millis() as i64; + match projection_store.evict_content_older_than(cutoff) { + Ok(0) => {} + Ok(n) => tracing::info!(evicted = n, "reddit display content TTL eviction"), + Err(e) => tracing::warn!(err = %e, "reddit content TTL eviction failed"), + } + } + }); +} + pub async fn rebuild_projection( cfg: &AppConfig, ) -> Result> { let event_log = EventLog::new(cfg.event_log_path.clone()); let store_path = format!("{}/store", cfg.data_dir); let db = durable::Db::open(std::path::Path::new(&store_path))?; - let entity_store = EntityStore::from_db(&db)?; let projection_store = ProjectionStore::from_db(&db)?; - entity_store.reset()?; projection_store.reset()?; let stats = event_log - .replay(|record| { - projection_apply::apply_records(&projection_store, &entity_store, &[record]) - }) + .replay(|record| projection_apply::apply_records(&projection_store, &[record])) .await?; let cursor = projection_store.last_applied_event_count()?; if cursor != stats.last_seq { @@ -132,7 +143,6 @@ pub struct AppState { pub cfg: Arc, pub event_log: Arc, pub view_log: Arc, - pub entity_store: EntityStore, pub projection_store: ProjectionStore, pub views: ViewStore, journal: JournalClient, @@ -145,7 +155,6 @@ impl AppState { let view_log = Arc::new(ViewLog::new(cfg.views_log_path.clone())); let store_path = format!("{}/store", cfg.data_dir); let db = durable::Db::open(std::path::Path::new(&store_path))?; - let entity_store = EntityStore::from_db(&db)?; let projection_store = ProjectionStore::from_db(&db)?; let views = ViewStore::from_db(&db)?; @@ -157,22 +166,25 @@ impl AppState { } views.spawn_worker(view_log.clone()); - catch_up_projection(&event_log, &entity_store, &projection_store).await?; + catch_up_projection(&event_log, &projection_store).await?; let next_seq = event_log.last_sequence().await? + 1; let journal = JournalClient::spawn( event_log.clone(), - entity_store.clone(), projection_store.clone(), next_seq, ); - let reddit = RedditBroker::spawn(journal.clone(), RedditApiConfig::from_env()); + let reddit = RedditBroker::spawn( + journal.clone(), + projection_store.clone(), + RedditApiConfig::from_env(), + ); + spawn_content_evictor(projection_store.clone()); Ok(Self { cfg: Arc::new(cfg), event_log, view_log, - entity_store, projection_store, views, journal, @@ -248,54 +260,72 @@ impl AppState { mod tests { use super::{normalize_scope, parse_item_param, AppConfig, AppState}; use crate::{ - entity_store::EntityStore, event_log::EventLog, events::Event, path_types::ItemId, - projection_apply, projection_store::ProjectionStore, + event_log::EventLog, events::Event, path_types::ItemId, projection_apply, + projection_store::ProjectionStore, reducer::EntityData, }; - use serde_json::json; fn event_record(seq: u64, event: Event) -> crate::events::EventRecord { crate::events::EventRecord::new(seq, crate::events::event_timestamp(&event), event) } #[tokio::test] - async fn replay_entity_imported_restores_view() { + async fn rebuild_projection_drops_ephemeral_content() { let tmp = tempfile::tempdir().unwrap(); - let log_path = tmp.path().join("events.jsonl"); - let log = EventLog::new(log_path.to_string_lossy().into_owned()); - let payload = json!({"kind":"t5","data":{"title":"Rust","display_name":"rust"}}); - let event = Event::EntityImported { - id: "https://reddit.com/r/rust".into(), - ts: 1, - payload: payload.clone(), - }; - log.append(&event_record(1, event)).await.unwrap(); + let data_dir = tmp.path().to_string_lossy().into_owned(); + let log = EventLog::new(format!("{data_dir}/events.jsonl")); + log.append(&event_record( + 1, + Event::NodeEnsured { + id: "https://reddit.com/r/rust".into(), + }, + )) + .await + .unwrap(); let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - super::catch_up_projection(&log, &entity_store, &projection_store) + super::catch_up_projection(&log, &projection_store) .await .unwrap(); - let tree = projection_store - .scope_tree(&ItemId::parse("https://reddit.com/r/rust").unwrap()) - .unwrap(); - let node = tree - .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) - .unwrap(); - assert_eq!(node.data.as_ref().unwrap().title, "Rust"); - let stored = entity_store - .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) - .unwrap() + let id = ItemId::parse("https://reddit.com/r/rust").unwrap(); + projection_store + .put_ephemeral_content( + &id, + &EntityData { + title: "Rust".into(), + author: None, + body_html: None, + thumb_url: None, + image_url: None, + link_url: None, + }, + 1, + ) .unwrap(); - assert_eq!(stored["data"]["display_name"], "rust"); + assert!(projection_store.load_node(&id).unwrap().unwrap().data.is_some()); + drop(projection_store); + drop(db); + + super::rebuild_projection(&AppConfig { + data_dir: data_dir.clone(), + event_log_path: format!("{data_dir}/events.jsonl"), + views_log_path: format!("{data_dir}/views.jsonl"), + port: 0, + }) + .await + .unwrap(); + + let db = durable::Db::open(tmp.path().join("store")).unwrap(); + let projection_store = ProjectionStore::from_db(&db).unwrap(); + let node = projection_store.load_node(&id).unwrap().unwrap(); + assert!(node.data.is_none()); } #[tokio::test] - async fn rebuild_projection_restores_nodes_payloads_and_cursor_from_jsonl() { + async fn rebuild_projection_restores_structure_and_cursor_from_jsonl() { let tmp = tempfile::tempdir().unwrap(); let data_dir = tmp.path().to_string_lossy().into_owned(); let log = EventLog::new(format!("{data_dir}/events.jsonl")); - let payload = json!({"kind":"t5","data":{"title":"Rust","display_name":"rust"}}); log.append_batch(&[ event_record( 1, @@ -305,16 +335,8 @@ mod tests { ), event_record( 2, - Event::EntityImported { - id: "https://reddit.com/r/rust".into(), - ts: 2, - payload: payload.clone(), - }, - ), - event_record( - 3, Event::VoteRecorded { - ts: 3, + ts: 2, a: "alpha".into(), b: "beta".into(), ratio_left: 2, @@ -328,11 +350,9 @@ mod tests { { let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); projection_apply::apply_records( &projection_store, - &entity_store, &[event_record( 1, Event::NodeEnsured { @@ -352,13 +372,12 @@ mod tests { }) .await .unwrap(); - assert_eq!(stats.applied, 3); - assert_eq!(stats.last_seq, 3); + assert_eq!(stats.applied, 2); + assert_eq!(stats.last_seq, 2); let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - assert_eq!(projection_store.last_applied_event_count().unwrap(), 3); + assert_eq!(projection_store.last_applied_event_count().unwrap(), 2); let tree = projection_store.scope_tree(&ItemId::root()).unwrap(); let root = tree.get(&ItemId::root()).unwrap(); assert!(root.children.contains(&ItemId::parse("alpha").unwrap())); @@ -366,11 +385,6 @@ mod tests { .load_node(&ItemId::parse("https://reddit.com/r/stale").unwrap()) .unwrap() .is_none()); - let stored = entity_store - .get(&ItemId::parse("https://reddit.com/r/rust").unwrap()) - .unwrap() - .unwrap(); - assert_eq!(stored["data"]["display_name"], "rust"); } #[tokio::test] @@ -389,12 +403,9 @@ mod tests { { let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - // Advance the projection cursor to 2 while the log tail is only 1. projection_apply::apply_records( &projection_store, - &entity_store, &[event_record( 2, Event::NodeEnsured { @@ -403,7 +414,7 @@ mod tests { )], ) .unwrap(); - let err = super::catch_up_projection(&log, &entity_store, &projection_store) + let err = super::catch_up_projection(&log, &projection_store) .await .unwrap_err(); assert!(err @@ -432,10 +443,9 @@ mod tests { .unwrap(); let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - super::catch_up_projection(&log, &entity_store, &projection_store) + super::catch_up_projection(&log, &projection_store) .await .unwrap(); assert_eq!(projection_store.last_applied_event_count().unwrap(), 1); @@ -444,7 +454,7 @@ mod tests { let first_edge_total: f64 = first_root.local_ranking.edges.values().sum(); assert_eq!(first_edge_total, 3.0); - super::catch_up_projection(&log, &entity_store, &projection_store) + super::catch_up_projection(&log, &projection_store) .await .unwrap(); assert_eq!(projection_store.last_applied_event_count().unwrap(), 1); @@ -556,9 +566,8 @@ mod tests { .unwrap(); { let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - super::catch_up_projection(&log, &entity_store, &projection_store) + super::catch_up_projection(&log, &projection_store) .await .unwrap(); } @@ -605,9 +614,8 @@ mod tests { .unwrap(); { let db = durable::Db::open(tmp.path().join("store")).unwrap(); - let entity_store = EntityStore::from_db(&db).unwrap(); let projection_store = ProjectionStore::from_db(&db).unwrap(); - super::catch_up_projection(&log, &entity_store, &projection_store) + super::catch_up_projection(&log, &projection_store) .await .unwrap(); } diff --git a/server/src/storage_dto.rs b/server/src/storage_dto.rs index de5ed995797ae306d3a71394a4d9d245ffd5771d..9dfb13c53efe4389277625a6ab3bfc18f566a453 100644 --- a/server/src/storage_dto.rs +++ b/server/src/storage_dto.rs @@ -3,17 +3,15 @@ //! Node structure (children, edges, voted pairs, recent votes) is no longer a //! single blob — it lives as point-addressable durable collections (see //! [`crate::storage_schema`]). This module only defines the small leaf values: -//! the derived entity view, raw entity payloads, and individual votes. +//! ephemeral entity views and individual votes. use serde::{Deserialize, Serialize}; -use serde_json::Value; use crate::{ path_types::ItemId, reducer::{EntityData, VoteData}, }; -pub const ENTITY_RECORD_VERSION: u32 = 1; pub const VOTE_RECORD_VERSION: u32 = 1; pub const ENTITY_DATA_VERSION: u32 = 1; @@ -29,14 +27,7 @@ impl Versioned { } } -pub type StoredEntityRecord = Versioned; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StoredEntityV1 { - pub json: Value, -} - -/// Derived entity view stored at a node's `data` leaf. +/// Derived entity view stored at a node's `data` leaf (ephemeral; not logged). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StoredEntityDataV1 { pub version: u32, @@ -63,25 +54,6 @@ pub struct StoredVoteV1 { pub thread_tag: String, } -pub fn encode_entity_payload(payload: &Value) -> StoredEntityRecord { - Versioned::new( - ENTITY_RECORD_VERSION, - StoredEntityV1 { - json: payload.clone(), - }, - ) -} - -pub fn decode_entity_payload(record: StoredEntityRecord) -> Result { - if record.version != ENTITY_RECORD_VERSION { - return Err(format!( - "unsupported entity record version: {}", - record.version - )); - } - Ok(record.payload.json) -} - pub fn encode_entity_data(data: &EntityData) -> StoredEntityDataV1 { StoredEntityDataV1 { version: ENTITY_DATA_VERSION, diff --git a/server/src/storage_schema.rs b/server/src/storage_schema.rs index 76bf7bc74a2c5ef4f78678a333b7661778f5b835..bd26e665e084b95b10fdfff091c31e8dc84d07b8 100644 --- a/server/src/storage_schema.rs +++ b/server/src/storage_schema.rs @@ -15,7 +15,7 @@ use crate::{ reducer::{EntityData, GroupState, NodeState, VoteData}, storage_dto::{ decode_entity_data, decode_vote, encode_entity_data, encode_vote, parse_stored_id, - StoredEntityDataV1, StoredEntityRecord, StoredVoteV1, + StoredEntityDataV1, StoredVoteV1, }, }; @@ -40,17 +40,17 @@ pub struct NodeSchema { pub voted_pairs: Map>, /// Recent votes, newest at the front (capped on write). pub recent_votes: Deque>, + /// When ephemeral Reddit display content was last fetched (ms); absent after eviction. + pub fetched_at: Leaf, } -/// The single database root: nodes, raw payloads, view counts, and per-concern -/// metadata maps (cursors and schema versions). +/// The single database root: nodes, view counts, and per-concern metadata maps +/// (cursors and schema versions). #[derive(Durable)] #[allow(dead_code)] pub struct Store { pub nodes: Map, pub proj_meta: Map>, - pub entities: Map>, - pub entity_meta: Map>, pub view_counts: Map>, pub view_meta: Map>, } @@ -264,12 +264,17 @@ pub fn vote_writes( Ok(()) } -/// Reified writes for an imported entity view (node data + path wiring). -pub fn entity_view_writes(batch: &mut Batch, id: &ItemId, view: Option<&EntityData>) { +/// Reified writes for ephemeral Reddit display content (not event-logged). +pub fn entity_content_writes(batch: &mut Batch, id: &ItemId, view: &EntityData, fetched_at: i64) { ensure_path_writes(batch, id); - if let Some(view) = view { - batch.write(node(id).data().set(&encode_entity_data(view))); - } + batch.write(node(id).data().set(&encode_entity_data(view))); + batch.write(node(id).fetched_at().set(&fetched_at)); +} + +/// Clear cached display content for one node (structure/votes are untouched). +pub fn entity_content_clear_writes(batch: &mut Batch, id: &ItemId) { + batch.write(node(id).data().delete()); + batch.write(node(id).fetched_at().delete()); } #[cfg(test)] diff --git a/test/reddit_import.clj b/test/reddit_import.clj index b476488526252c13fd73bdda76e5201678e4a714..6d8c5bca7ebad0b5abfddecd4ea48cc09d738ebe 100644 --- a/test/reddit_import.clj +++ b/test/reddit_import.clj @@ -59,9 +59,10 @@ "curl" "-sf" browse-url)) log (slurp (io/file log-path))] (is (str/includes? after "The Rust Programming Language")) - (is (str/includes? log "\"type\":\"entity_imported\"")) - (is (str/includes? log "\"subscribers\":350000")) - (is (str/includes? log "\"display_name\":\"rust\""))) + (is (str/includes? log "\"type\":\"node_ensured\"")) + (is (not (str/includes? log "\"subscribers\""))) + (is (not (str/includes? log "\"display_name\""))) + (is (not (str/includes? log "entity_imported")))) (let [children-sse (curl-fetch-ui-sse app-base "reddit.com/r/rust" "children")] (is (zero? (:exit children-sse)) "POST /ui fetch_entity (children) SSE succeeds") (is (str/includes? (:out children-sse) "Idiomorph.morph")) @@ -71,10 +72,12 @@ log2 (slurp (io/file log-path))] (is (str/includes? after-children "Announcing Rust 1.99")) (is (str/includes? after-children "Unranked")) - (is (str/includes? log2 "announcing_rust_199"))))) + (is (str/includes? log2 "\"type\":\"node_ensured\"")) + (is (str/includes? log2 "/comments/")) + (is (not (str/includes? log2 "\"selftext\"")))))) (deftest reddit-fetch-via-mock-api - (testing "Fetch more queues import; event log stores full payload; page shows title" + (testing "Fetch caches display content ephemerally; log records structure only" (let [root (repo-root) fixtures (mock-reddit/fixtures-dir root) data-dir (.getAbsolutePath