You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [15e1037a] url stuff Side A — unified diff (full patch): diff --git a/server/src/url_rules/graph.rs b/server/src/url_rules/graph.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7ac0f9a551a1727cb2f9294778c283b9885b147 --- /dev/null +++ b/server/src/url_rules/graph.rs @@ -0,0 +1,831 @@ +//! Semantic URL graph: DFA traversal on host + path, query in context, generic fallback. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use url::Url; + +use super::graph_builder::GraphBuilder; +use super::parse::{normalize_match_host, strip_tracking_query, UrlParts}; + +#[derive(Debug, Clone, Default)] +pub struct Context { + pub vars: HashMap, + pub query: HashMap, +} + +pub type CanonicalFn = fn(&Context) -> Option; + +#[derive(Clone, Copy)] +pub enum EdgePattern { + Literal(&'static str), + Variable(&'static str), + /// Absorb any trailing segment without leaving this node (e.g. post title slug). + AbsorbAny, + /// Absorb segment when `cond(seg)` (e.g. subreddit listing suffix). + AbsorbIf(fn(&str) -> bool), +} + +pub struct Edge { + pub pattern: EdgePattern, + pub target: &'static str, +} + +pub struct Node { + pub edges: Vec, + pub canonical: CanonicalFn, + pub parent: Option<&'static str>, +} + +impl Node { + pub(crate) fn empty() -> Self { + Self { + edges: Vec::new(), + canonical: |_| None, + parent: None, + } + } +} + +pub struct Graph { + pub nodes: HashMap<&'static str, Node>, +} + +static GRAPH: OnceLock = OnceLock::new(); + +pub fn graph() -> &'static Graph { + GRAPH.get_or_init(build_graph) +} + +impl Graph { + pub fn resolve_canonical(&self, parts: &UrlParts) -> Option { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + + if let Some(node_id) = self.traverse(parts, &mut ctx) { + if let Some(canon) = (self.nodes.get(node_id)?.canonical)(&ctx) { + return Some(canon); + } + } + Some(generic_canonical(parts)) + } + + pub fn breadcrumbs(&self, parts: &UrlParts) -> Vec { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + + if let Some(mut node_id) = self.traverse(parts, &mut ctx) { + let mut paths = Vec::new(); + loop { + let node = match self.nodes.get(node_id) { + Some(n) => n, + None => break, + }; + if let Some(url) = (node.canonical)(&ctx) { + if paths.last() != Some(&url) { + paths.push(url); + } + } + match node.parent { + Some(p) => node_id = p, + None => break, + } + } + paths.reverse(); + if !paths.is_empty() { + return paths; + } + } + generic_breadcrumbs(parts) + } + + fn traverse(&self, parts: &UrlParts, ctx: &mut Context) -> Option<&'static str> { + let host = parts.match_host(); + let mut node_id = match host.as_str() { + "reddit.com" => "reddit_root", + "youtube.com" => "youtube_root", + "youtu.be" => "youtu_be_entry", + _ => return None, + }; + + let segs: Vec<&str> = parts.path_segments.iter().map(String::as_str).collect(); + let mut i = 0; + while i < segs.len() { + let seg = segs[i]; + match self.follow_edge(node_id, seg, ctx) { + Ok(next) => { + node_id = next; + i += 1; + } + Err(()) => { + if self.try_absorb(node_id, seg) { + i += 1; + continue; + } + return None; + } + } + } + Some(node_id) + } + + fn follow_edge( + &self, + node_id: &'static str, + seg: &str, + ctx: &mut Context, + ) -> Result<&'static str, ()> { + let node = self.nodes.get(node_id).ok_or(())?; + for edge in &node.edges { + match edge.pattern { + EdgePattern::Literal(lit) if lit == seg => return Ok(edge.target), + EdgePattern::Variable(name) => { + ctx.vars.insert(name.to_string(), seg.to_string()); + return Ok(edge.target); + } + EdgePattern::AbsorbAny + | EdgePattern::AbsorbIf(_) + | EdgePattern::Literal(_) + | EdgePattern::Variable(_) => {} + } + } + Err(()) + } + + fn try_absorb(&self, node_id: &'static str, seg: &str) -> bool { + let node = match self.nodes.get(node_id) { + Some(n) => n, + None => return false, + }; + for edge in &node.edges { + match edge.pattern { + EdgePattern::AbsorbAny => return true, + EdgePattern::AbsorbIf(cond) if cond(seg) => return true, + EdgePattern::AbsorbIf(_) | EdgePattern::Literal(_) | EdgePattern::Variable(_) => {} + } + } + false + } + + /// Test hook: terminal graph node and captured context after traversal. + #[cfg(test)] + pub fn traverse_terminal(&self, parts: &UrlParts) -> Option<(&'static str, Context)> { + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + let mut ctx = Context { + vars: HashMap::new(), + query, + }; + let node = self.traverse(parts, &mut ctx)?; + Some((node, ctx)) + } +} + +fn is_reddit_listing_suffix(seg: &str) -> bool { + matches!(seg, "hot" | "top" | "new" | "rising" | "controversial") +} + +/// Percent-encode a path or query fragment so `&`, `?`, etc. cannot break URL structure. +fn enc(s: &str) -> String { + urlencoding::encode(s).into_owned() +} + +// --- Canonical formatters --- + +fn canon_reddit_root(_: &Context) -> Option { + Some("https://reddit.com".to_string()) +} + +fn canon_reddit_r_hub(_: &Context) -> Option { + Some("https://reddit.com/r".to_string()) +} + +fn canon_reddit_subreddit(ctx: &Context) -> Option { + let sub = ctx.vars.get("subreddit")?; + Some(format!( + "https://reddit.com/r/{}", + enc(&sub.to_ascii_lowercase()) + )) +} + +fn canon_reddit_post(ctx: &Context) -> Option { + let sub = ctx.vars.get("subreddit")?.to_ascii_lowercase(); + let id = ctx.vars.get("post_id")?; + Some(format!( + "https://reddit.com/r/{}/comments/{}", + enc(&sub), + enc(id) + )) +} + +fn canon_youtube_root(_: &Context) -> Option { + Some("https://youtube.com".to_string()) +} + +fn canon_youtube_watch(ctx: &Context) -> Option { + let v = ctx + .query + .get("v") + .or_else(|| ctx.vars.get("video_id"))?; + Some(format!("https://youtube.com/watch?v={}", enc(v))) +} + +fn canon_youtu_be(ctx: &Context) -> Option { + let v = ctx.vars.get("vid_id")?; + Some(format!("https://youtube.com/watch?v={}", enc(v))) +} + +pub fn build_graph() -> Graph { + GraphBuilder::new() + .node("reddit_root") + .canonical(canon_reddit_root) + .edge(EdgePattern::Literal("r"), "reddit_r_hub") + .node("reddit_r_hub") + .parent("reddit_root") + .canonical(canon_reddit_r_hub) + .edge(EdgePattern::Variable("subreddit"), "reddit_subreddit") + .node("reddit_subreddit") + .parent("reddit_r_hub") + .canonical(canon_reddit_subreddit) + .edge( + EdgePattern::AbsorbIf(is_reddit_listing_suffix), + "reddit_subreddit", + ) + .edge(EdgePattern::Literal("comments"), "reddit_comments_gate") + .node("reddit_comments_gate") + .parent("reddit_subreddit") + .canonical(canon_reddit_subreddit) + .edge(EdgePattern::Variable("post_id"), "reddit_post") + .node("reddit_post") + .parent("reddit_subreddit") + .canonical(canon_reddit_post) + .edge(EdgePattern::AbsorbAny, "reddit_post") + .node("youtube_root") + .canonical(canon_youtube_root) + .edge(EdgePattern::Literal("watch"), "youtube_watch") + .edge(EdgePattern::Literal("shorts"), "youtube_shorts_gate") + .node("youtube_watch") + .parent("youtube_root") + .canonical(canon_youtube_watch) + .node("youtube_shorts_gate") + .parent("youtube_root") + .canonical(canon_youtube_root) + .edge(EdgePattern::Variable("video_id"), "youtube_watch") + .node("youtu_be_entry") + .canonical(canon_youtube_root) + .edge(EdgePattern::Variable("vid_id"), "youtu_be_video") + .node("youtu_be_video") + .parent("youtube_root") + .canonical(canon_youtu_be) + .build() +} + +// --- Generic internet fallback --- + +pub fn generic_canonical(parts: &UrlParts) -> String { + let host = normalize_match_host(&parts.host); + let path_segments: Vec = parts.path_segments.clone(); + let mut query = parts.query.clone(); + strip_tracking_query(&mut query); + + let mut url = if path_segments.is_empty() { + Url::parse(&format!("https://{host}")) + .unwrap_or_else(|_| Url::parse("https://invalid").unwrap()) + } else { + let path = format!("/{}", path_segments.join("/")); + Url::parse(&format!("https://{host}{path}")) + .unwrap_or_else(|_| Url::parse("https://invalid").unwrap()) + }; + + if !query.is_empty() { + let mut pairs: Vec<_> = query.iter().collect(); + pairs.sort_by(|a, b| a.0.cmp(b.0)); + url.query_pairs_mut().clear(); + for (k, v) in pairs { + url.query_pairs_mut().append_pair(k, v); + } + } + + let mut s = url.to_string(); + if path_segments.is_empty() { + s = s.trim_end_matches('/').to_string(); + } + s +} + +pub fn generic_breadcrumbs(parts: &UrlParts) -> Vec { + let host = normalize_match_host(&parts.host); + let n = parts.path_segments.len(); + let mut out = Vec::new(); + + let base = generic_canonical(&UrlParts { + scheme: "https".to_string(), + host: host.clone(), + path_segments: vec![], + query: HashMap::new(), + }); + out.push(base); + + for i in 0..n { + let segs: Vec = parts.path_segments[..=i].to_vec(); + let url = generic_canonical(&UrlParts { + scheme: "https".to_string(), + host: host.clone(), + path_segments: segs, + query: HashMap::new(), + }); + if out.last() != Some(&url) { + out.push(url); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::url_rules::parse::test_parts; + + fn g() -> &'static Graph { + graph() + } + + fn canon(parts: &UrlParts) -> String { + g().resolve_canonical(parts).unwrap() + } + + fn crumbs(parts: &UrlParts) -> Vec { + g().breadcrumbs(parts) + } + + fn terminal(parts: &UrlParts) -> Option<&'static str> { + g().traverse_terminal(parts).map(|(n, _)| n) + } + + fn vars(parts: &UrlParts) -> HashMap { + g().traverse_terminal(parts) + .map(|(_, c)| c.vars) + .unwrap_or_default() + } + + #[test] + fn youtu_be_malicious_segment_encoded_not_injected() { + let p = test_parts("youtu.be", &["abc&t=1"], &[]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=abc%26t%3D1"); + assert!(!canon(&p).contains("abc&t=1")); + } + + #[test] + fn youtube_query_v_encoded() { + let p = test_parts("youtube.com", &["watch"], &[("v", "a&b=c")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=a%26b%3Dc"); + } + + #[test] + fn absorb_patterns_live_on_edges_not_in_engine() { + let g = build_graph(); + let sub = g.nodes.get("reddit_subreddit").unwrap(); + assert!(sub + .edges + .iter() + .any(|e| matches!(e.pattern, EdgePattern::AbsorbIf(_)))); + let post = g.nodes.get("reddit_post").unwrap(); + assert!(post + .edges + .iter() + .any(|e| matches!(e.pattern, EdgePattern::AbsorbAny))); + } + + #[test] + fn builder_rejects_missing_parent() { + let result = std::panic::catch_unwind(|| { + GraphBuilder::new() + .node("orphan") + .parent("nonexistent_parent") + .build(); + }); + assert!(result.is_err()); + } + + #[test] + fn generic_canonical_forces_https_and_strips_www() { + let p = test_parts("www.example.com", &["blog", "post"], &[]); + assert_eq!(canon(&p), "https://example.com/blog/post"); + } + + #[test] + fn generic_canonical_sorts_query_keys() { + let p = test_parts("example.com", &["search"], &[("q", "rust"), ("page", "2")]); + assert_eq!(canon(&p), "https://example.com/search?page=2&q=rust"); + } + + #[test] + fn generic_canonical_strips_tracking_from_query() { + let p = test_parts( + "news.ycombinator.com", + &["item"], + &[("id", "1"), ("utm_medium", "social")], + ); + assert_eq!(canon(&p), "https://news.ycombinator.com/item?id=1"); + } + + #[test] + fn generic_breadcrumbs_cumulative_path() { + let p = test_parts("paulgraham.com", &["articles", "lisp.html"], &[]); + assert_eq!( + crumbs(&p), + vec![ + "https://paulgraham.com", + "https://paulgraham.com/articles", + "https://paulgraham.com/articles/lisp.html" + ] + ); + } + + #[test] + fn generic_breadcrumbs_domain_only() { + let p = test_parts("example.com", &[], &[]); + assert_eq!(crumbs(&p), vec!["https://example.com"]); + } + + #[test] + fn unknown_host_uses_generic_not_graph() { + let p = test_parts("hackernews.com", &["item", "123"], &[]); + assert_eq!(terminal(&p), None); + assert_eq!(canon(&p), "https://hackernews.com/item/123"); + } + + #[test] + fn traverse_captures_subreddit_variable() { + let p = test_parts("reddit.com", &["r", "Rust"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(vars(&p).get("subreddit").map(String::as_str), Some("Rust")); + } + + #[test] + fn traverse_captures_post_id() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "abc123"], &[]); + assert_eq!(terminal(&p), Some("reddit_post")); + assert_eq!(vars(&p).get("post_id").map(String::as_str), Some("abc123")); + } + + #[test] + fn traverse_absorbs_listing_suffix_stays_on_subreddit() { + let p = test_parts("reddit.com", &["r", "rust", "hot"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn traverse_absorbs_all_listing_suffixes() { + for suffix in ["hot", "top", "new", "rising", "controversial"] { + let p = test_parts("reddit.com", &["r", "test", suffix], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit"), "suffix {suffix}"); + assert_eq!(canon(&p), "https://reddit.com/r/test", "suffix {suffix}"); + } + } + + #[test] + fn traverse_absorbs_post_title_slug() { + let p = test_parts( + "reddit.com", + &["r", "rust", "comments", "aaa", "my_great_post_title"], + &[], + ); + assert_eq!(terminal(&p), Some("reddit_post")); + assert_eq!(canon(&p), "https://reddit.com/r/rust/comments/aaa"); + } + + #[test] + fn traverse_unknown_segment_falls_back_to_generic() { + let p = test_parts("reddit.com", &["r", "rust", "wiki", "faq"], &[]); + assert_eq!(terminal(&p), None); + assert_eq!(canon(&p), "https://reddit.com/r/rust/wiki/faq"); + } + + #[test] + fn traverse_youtube_watch_requires_v_in_query() { + let p = test_parts("youtube.com", &["watch"], &[("v", "xyz")]); + assert_eq!(terminal(&p), Some("youtube_watch")); + } + + #[test] + fn traverse_youtu_be_captures_vid_id() { + let p = test_parts("youtu.be", &["dQw4w9WgXcQ"], &[]); + assert_eq!(terminal(&p), Some("youtu_be_video")); + assert_eq!(vars(&p).get("vid_id").map(String::as_str), Some("dQw4w9WgXcQ")); + } + + #[test] + fn traverse_shorts_sets_video_id_var() { + let p = test_parts("youtube.com", &["shorts", "abc99"], &[]); + assert_eq!(terminal(&p), Some("youtube_watch")); + assert_eq!(vars(&p).get("video_id").map(String::as_str), Some("abc99")); + } + + #[test] + fn reddit_domain_canonical() { + let p = test_parts("reddit.com", &[], &[]); + assert_eq!(canon(&p), "https://reddit.com"); + } + + #[test] + fn reddit_r_hub_canonical() { + let p = test_parts("reddit.com", &["r"], &[]); + assert_eq!(terminal(&p), Some("reddit_r_hub")); + assert_eq!(canon(&p), "https://reddit.com/r"); + } + + #[test] + fn reddit_subreddit_lowercases_name() { + let p = test_parts("reddit.com", &["r", "AmITheAsshole"], &[]); + assert_eq!(canon(&p), "https://reddit.com/r/amitheasshole"); + } + + #[test] + fn reddit_host_aliases_old_new_www() { + for host in ["old.reddit.com", "new.reddit.com", "www.reddit.com"] { + let p = test_parts(host, &["r", "rust"], &[]); + assert_eq!(canon(&p), "https://reddit.com/r/rust", "host {host}"); + } + } + + #[test] + fn reddit_post_strips_slug_and_query() { + let p = test_parts( + "old.reddit.com", + &["r", "Rust", "comments", "1abc", "title_slug_here"], + &[("sort", "new")], + ); + assert_eq!(canon(&p), "https://reddit.com/r/rust/comments/1abc"); + } + + #[test] + fn reddit_post_multiple_slugs_absorbed() { + let p = test_parts( + "reddit.com", + &["r", "x", "comments", "id1", "slug1", "extra"], + &[], + ); + assert_eq!(canon(&p), "https://reddit.com/r/x/comments/id1"); + } + + #[test] + fn reddit_listing_with_query_only() { + let p = test_parts("www.reddit.com", &["r", "programming"], &[("sort", "top")]); + assert_eq!(canon(&p), "https://reddit.com/r/programming"); + } + + #[test] + fn reddit_subreddit_breadcrumbs_include_r_hub() { + let p = test_parts("reddit.com", &["r", "movies"], &[]); + assert_eq!( + crumbs(&p), + vec![ + "https://reddit.com", + "https://reddit.com/r", + "https://reddit.com/r/movies" + ] + ); + } + + #[test] + fn reddit_post_breadcrumbs_skip_comments_node() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "1trnvdl"], &[]); + let c = crumbs(&p); + assert!(!c.iter().any(|u| u.ends_with("/comments"))); + assert_eq!( + c.last().map(String::as_str), + Some("https://reddit.com/r/aww/comments/1trnvdl") + ); + assert!(c.contains(&"https://reddit.com/r/aww".to_string())); + } + + #[test] + fn reddit_post_parent_is_subreddit_not_comments() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "1trnvdl"], &[]); + let c = crumbs(&p); + let parent = c.get(c.len() - 2).unwrap(); + assert_eq!(parent, "https://reddit.com/r/aww"); + } + + #[test] + fn reddit_domain_parent_is_none_in_breadcrumb_chain() { + let p = test_parts("reddit.com", &[], &[]); + assert_eq!(crumbs(&p), vec!["https://reddit.com"]); + } + + #[test] + fn youtube_watch_canonical_uses_v_only() { + let p = test_parts("youtube.com", &["watch"], &[("v", "abc"), ("t", "99")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=abc"); + } + + #[test] + fn youtube_query_order_independent() { + let a = test_parts("youtube.com", &["watch"], &[("v", "abc"), ("t", "4")]); + let b = test_parts("youtube.com", &["watch"], &[("t", "4"), ("v", "abc")]); + assert_eq!(canon(&a), canon(&b)); + } + + #[test] + fn youtube_host_aliases() { + for host in ["www.youtube.com", "m.youtube.com"] { + let p = test_parts(host, &["watch"], &[("v", "x")]); + assert_eq!(canon(&p), "https://youtube.com/watch?v=x", "host {host}"); + } + } + + #[test] + fn youtube_shorts_canonical_matches_watch() { + let shorts = test_parts("youtube.com", &["shorts", "vid123"], &[]); + let watch = test_parts("youtube.com", &["watch"], &[("v", "vid123")]); + assert_eq!(canon(&shorts), canon(&watch)); + assert_eq!(canon(&shorts), "https://youtube.com/watch?v=vid123"); + } + + #[test] + fn youtu_be_matches_youtube_watch() { + let be = test_parts("youtu.be", &["dQw4w9WgXcQ"], &[]); + let watch = test_parts("youtube.com", &["watch"], &[("v", "dQw4w9WgXcQ")]); + assert_eq!(canon(&be), canon(&watch)); + } + + #[test] + fn youtube_breadcrumbs_domain_then_watch() { + let p = test_parts("youtube.com", &["watch"], &[("v", "abc")]); + assert_eq!( + crumbs(&p), + vec!["https://youtube.com", "https://youtube.com/watch?v=abc"] + ); + } + + #[test] + fn youtu_be_breadcrumbs_include_youtube_domain() { + let p = test_parts("youtu.be", &["abc"], &[]); + let c = crumbs(&p); + assert_eq!(c.first().map(String::as_str), Some("https://youtube.com")); + assert_eq!( + c.last().map(String::as_str), + Some("https://youtube.com/watch?v=abc") + ); + } + + #[test] + fn parsed_urls_match_hand_built_parts() { + let raw = "https://www.reddit.com/r/rust/comments/aaa/title/?utm=x"; + let parsed = UrlParts::parse(raw).unwrap(); + let hand = test_parts( + "www.reddit.com", + &["r", "rust", "comments", "aaa", "title"], + &[("utm", "x")], + ); + assert_eq!(canon(&parsed), canon(&hand)); + } + + #[test] + fn equivalence_cluster_youtube_formats() { + let urls = [ + "https://youtu.be/abc123", + "https://www.youtube.com/watch?v=abc123", + "https://youtube.com/watch?v=abc123&t=1", + "https://m.youtube.com/watch?t=1&v=abc123", + ]; + let canonical: Vec<_> = urls + .iter() + .map(|u| canon(&UrlParts::parse(u).unwrap())) + .collect(); + assert!(canonical.iter().all(|c| *c == "https://youtube.com/watch?v=abc123")); + } + + #[test] + fn equivalence_cluster_reddit_post_formats() { + let urls = [ + "https://old.reddit.com/r/Rust/comments/aaa/slug/", + "reddit.com/r/rust/comments/aaa/other_slug", + "https://reddit.com/r/RUST/comments/aaa", + ]; + let canonical: Vec<_> = urls + .iter() + .map(|u| canon(&UrlParts::parse(u).unwrap())) + .collect(); + assert!( + canonical + .iter() + .all(|c| *c == "https://reddit.com/r/rust/comments/aaa") + ); + } + + #[test] + fn graph_nodes_all_have_valid_parent_links() { + let g = build_graph(); + for (id, node) in &g.nodes { + if let Some(parent) = node.parent { + assert!(g.nodes.contains_key(parent), "node {id} parent {parent}"); + } + } + } + + #[test] + fn graph_terminal_canonical_always_succeeds_for_reddit_paths() { + let cases: &[(&[&str], &str)] = &[ + (&["r", "rust"], "https://reddit.com/r/rust"), + ( + &["r", "rust", "comments", "x"], + "https://reddit.com/r/rust/comments/x", + ), + ]; + for (segs, want) in cases { + let p = test_parts("reddit.com", segs, &[]); + assert_eq!(canon(&p), *want); + } + } + + #[test] + fn breadcrumb_parent_walk_matches_parent_url_semantics() { + let p = test_parts("reddit.com", &["r", "aww", "comments", "id1"], &[]); + let c = crumbs(&p); + assert_eq!(c.len(), 4); + assert_eq!( + c.get(c.len() - 2).map(String::as_str), + Some("https://reddit.com/r/aww") + ); + } + + #[test] + fn youtube_watch_without_v_falls_back_to_generic() { + let p = test_parts("youtube.com", &["watch"], &[]); + assert_eq!(terminal(&p), Some("youtube_watch")); + assert_eq!(canon(&p), "https://youtube.com/watch"); + } + + #[test] + fn reddit_only_comments_path_stops_at_gate() { + let p = test_parts("reddit.com", &["r", "rust", "comments"], &[]); + assert_eq!(terminal(&p), Some("reddit_comments_gate")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn generic_deep_path_many_segments() { + let segs: Vec<&str> = (0..10) + .map(|i| match i { + 0 => "a", + 1 => "b", + 2 => "c", + 3 => "d", + 4 => "e", + 5 => "f", + 6 => "g", + 7 => "h", + 8 => "i", + _ => "j", + }) + .collect(); + let p = test_parts("site.com", &segs, &[]); + assert_eq!(crumbs(&p).len(), 11); + } + + #[test] + fn traverse_literal_r_required_for_subreddit() { + let p = test_parts("reddit.com", &["rust"], &[]); + assert_eq!(terminal(&p), None); + } + + #[test] + fn http_scheme_upgraded_via_generic_fallback_host() { + let parsed = UrlParts::parse("http://example.com/page").unwrap(); + assert_eq!(canon(&parsed), "https://example.com/page"); + } + + #[test] + fn each_graph_node_canonical_is_invokable() { + let g = build_graph(); + let empty = Context::default(); + for (id, node) in &g.nodes { + let _ = (node.canonical)(&empty); + let _ = id; + } + } + + #[test] + fn reddit_double_listing_suffix_both_absorbed() { + let p = test_parts("reddit.com", &["r", "rust", "hot", "new"], &[]); + assert_eq!(terminal(&p), Some("reddit_subreddit")); + assert_eq!(canon(&p), "https://reddit.com/r/rust"); + } + + #[test] + fn youtu_be_empty_path_stays_at_entry() { + let p = test_parts("youtu.be", &[], &[]); + assert_eq!(terminal(&p), Some("youtu_be_entry")); + } +} diff --git a/server/src/url_rules/graph_builder.rs b/server/src/url_rules/graph_builder.rs new file mode 100644 index 0000000000000000000000000000000000000000..243204ad5ca514fff459057956080cacb5bf38e2 --- /dev/null +++ b/server/src/url_rules/graph_builder.rs @@ -0,0 +1,73 @@ +//! Declarative construction of the URL graph with build-time link validation. + +use std::collections::HashMap; + +use super::graph::{CanonicalFn, Edge, EdgePattern, Graph, Node}; + +pub struct GraphBuilder { + nodes: HashMap<&'static str, Node>, + current: Option<&'static str>, +} + +impl GraphBuilder { + pub fn new() -> Self { + Self { + nodes: HashMap::new(), + current: None, + } + } + + pub fn node(mut self, id: &'static str) -> Self { + self.nodes.entry(id).or_insert_with(Node::empty); + self.current = Some(id); + self + } + + pub fn canonical(mut self, f: CanonicalFn) -> Self { + let id = self.current.expect("canonical() without node()"); + self.nodes.get_mut(id).expect("node missing").canonical = f; + self + } + + pub fn parent(mut self, parent_id: &'static str) -> Self { + let id = self.current.expect("parent() without node()"); + self.nodes.get_mut(id).expect("node missing").parent = Some(parent_id); + self + } + + pub fn edge(mut self, pattern: EdgePattern, target: &'static str) -> Self { + let id = self.current.expect("edge() without node()"); + self.nodes + .get_mut(id) + .expect("node missing") + .edges + .push(Edge { pattern, target }); + self + } + + pub fn build(self) -> Graph { + for (id, node) in &self.nodes { + if let Some(parent) = node.parent { + assert!( + self.nodes.contains_key(parent), + "node {id}: parent {parent} does not exist" + ); + } + for edge in &node.edges { + if !matches!( + edge.pattern, + EdgePattern::AbsorbAny | EdgePattern::AbsorbIf(_) + ) { + assert!( + self.nodes.contains_key(edge.target), + "node {id}: edge target {} does not exist", + edge.target + ); + } + } + } + Graph { + nodes: self.nodes, + } + } +} diff --git a/server/src/url_rules/mod.rs b/server/src/url_rules/mod.rs index 9e1445346ce77a49dd6a7e7713bf9c57aef353cc..ba4ac662acc7bd4d50ee34613eb7eb6fccbfb17b 100644 --- a/server/src/url_rules/mod.rs +++ b/server/src/url_rules/mod.rs @@ -1,6 +1,7 @@ //! URL canonicalization and hierarchy via a semantic graph (DFA + generic fallback). mod graph; +mod graph_builder; mod parse; mod registry; diff --git a/server/src/url_rules/parse.rs b/server/src/url_rules/parse.rs new file mode 100644 index 0000000000000000000000000000000000000000..19d0c82feb718817168b8b445fcbfa39cf6aa6ba --- /dev/null +++ b/server/src/url_rules/parse.rs @@ -0,0 +1,169 @@ +//! Parse raw strings into host, path segments, and query (order-independent). + +use std::collections::HashMap; + +use url::Url; + +#[derive(Debug, Clone)] +pub struct UrlParts { + pub scheme: String, + pub host: String, + pub path_segments: Vec, + pub query: HashMap, +} + +impl UrlParts { + pub fn parse(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + let with_scheme = if trimmed.contains("://") { + trimmed.to_string() + } else if trimmed.starts_with("r/") || trimmed.starts_with("/r/") { + let rest = trimmed.trim_start_matches('/').trim_start_matches("r/"); + format!("https://reddit.com/r/{rest}") + } else if trimmed.contains('.') && !trimmed.starts_with('/') { + format!("https://{trimmed}") + } else { + trimmed.to_string() + }; + + let url = Url::parse(&with_scheme).ok()?; + let host = url.host_str()?.to_string(); + let path_segments: Vec = url + .path_segments() + .map(|segs| segs.filter(|s| !s.is_empty()).map(str::to_string).collect()) + .unwrap_or_default(); + + let mut query = HashMap::new(); + for (k, v) in url.query_pairs() { + query.insert(k.into_owned(), v.into_owned()); + } + + Some(Self { + scheme: url.scheme().to_string(), + path_segments, + query, + host, + }) + } + + /// Host normalized for graph entry matching (lowercase, aliases). + pub fn match_host(&self) -> String { + normalize_match_host(&self.host) + } +} + +pub fn normalize_match_host(host: &str) -> String { + let h = host + .strip_prefix("www.") + .unwrap_or(host) + .to_ascii_lowercase(); + match h.as_str() { + "old.reddit.com" | "new.reddit.com" => "reddit.com".to_string(), + "m.youtube.com" => "youtube.com".to_string(), + _ => h, + } +} + +pub fn strip_tracking_query(query: &mut HashMap) { + query.retain(|k, _| { + let lower = k.to_ascii_lowercase(); + !(lower.starts_with("utm_") + || matches!( + lower.as_str(), + "fbclid" | "gclid" | "ref" | "ref_src" | "ref_source" | "mc_cid" | "mc_eid" + )) + }); +} + +#[cfg(test)] +pub(crate) fn test_parts(host: &str, segs: &[&str], query: &[(&str, &str)]) -> UrlParts { + UrlParts { + scheme: "https".to_string(), + host: host.to_string(), + path_segments: segs.iter().map(|s| (*s).to_string()).collect(), + query: query + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_full_url_splits_host_path_query() { + let p = UrlParts::parse("https://www.youtube.com/watch?v=abc&t=4").unwrap(); + assert_eq!(p.host, "www.youtube.com"); + assert_eq!(p.path_segments, vec!["watch"]); + assert_eq!(p.query.get("v").map(String::as_str), Some("abc")); + assert_eq!(p.query.get("t").map(String::as_str), Some("4")); + } + + #[test] + fn parse_r_shortcut_expands_to_reddit() { + let p = UrlParts::parse("r/rust").unwrap(); + assert_eq!(p.match_host(), "reddit.com"); + assert_eq!(p.path_segments, vec!["r", "rust"]); + } + + #[test] + fn parse_slash_r_shortcut() { + let p = UrlParts::parse("/r/aww").unwrap(); + assert_eq!(p.path_segments, vec!["r", "aww"]); + } + + #[test] + fn parse_schemeless_host_path() { + let p = UrlParts::parse("reddit.com/r/rust/comments/aaa/slug").unwrap(); + assert_eq!(p.match_host(), "reddit.com"); + assert_eq!( + p.path_segments, + vec!["r", "rust", "comments", "aaa", "slug"] + ); + } + + #[test] + fn parse_empty_returns_none() { + assert!(UrlParts::parse("").is_none()); + assert!(UrlParts::parse(" ").is_none()); + } + + #[test] + fn normalize_match_host_reddit_aliases() { + assert_eq!(normalize_match_host("old.reddit.com"), "reddit.com"); + assert_eq!(normalize_match_host("NEW.reddit.com"), "reddit.com"); + assert_eq!(normalize_match_host("www.reddit.com"), "reddit.com"); + } + + #[test] + fn normalize_match_host_youtube_aliases() { + assert_eq!(normalize_match_host("m.youtube.com"), "youtube.com"); + assert_eq!(normalize_match_host("www.youtube.com"), "youtube.com"); + } + + #[test] + fn strip_tracking_query_removes_known_params() { + let mut q = HashMap::from([ + ("v".into(), "1".into()), + ("utm_source".into(), "x".into()), + ("fbclid".into(), "y".into()), + ("ref".into(), "z".into()), + ]); + strip_tracking_query(&mut q); + assert_eq!(q.len(), 1); + assert_eq!(q.get("v").map(String::as_str), Some("1")); + } + + #[test] + fn strip_tracking_query_utm_prefix() { + let mut q = HashMap::from([("utm_campaign".into(), "email".into())]); + strip_tracking_query(&mut q); + assert!(q.is_empty()); + } +} diff --git a/server/src/url_rules/registry_tests.rs b/server/src/url_rules/registry_tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..7b76b550f808f590e011469fdae02adf603b43d2 --- /dev/null +++ b/server/src/url_rules/registry_tests.rs @@ -0,0 +1,195 @@ +//! End-to-end tests for the public registry API (`canonicalize_raw`, breadcrumbs, parent). + +use super::registry::{ + canonicalize_raw, looks_like_url, navigable_breadcrumbs, parent_url, resolve_id, +}; + +fn canon(raw: &str) -> String { + canonicalize_raw(raw).unwrap().canonical +} + +#[test] +fn looks_like_url_positive_cases() { + for raw in [ + "https://reddit.com/r/rust", + "r/rust", + "/r/aww", + "reddit.com/r/x", + "www.example.com/path", + "youtu.be/abc", + ] { + assert!(looks_like_url(raw), "{raw}"); + } +} + +#[test] +fn looks_like_url_negative_cases() { + for raw in ["alpha", "beta", "", "hello world", "no-dots"] { + assert!(!looks_like_url(raw), "{raw}"); + } +} + +#[test] +fn resolve_id_matches_canonicalize_raw() { + let raw = "https://youtu.be/xyz"; + assert_eq!( + resolve_id(raw).as_deref(), + Some(canon(raw).as_str()) + ); +} + +#[test] +fn canonicalize_empty_returns_none() { + assert!(canonicalize_raw("").is_none()); +} + +#[test] +fn alias_when_slug_stripped() { + let r = canonicalize_raw( + "https://reddit.com/r/rust/comments/aaa/very_long_title_slug", + ) + .unwrap(); + assert_eq!(r.canonical, "https://reddit.com/r/rust/comments/aaa"); + assert!(r.alias_of.is_some()); +} + +#[test] +fn alias_none_when_already_canonical() { + let raw = "https://reddit.com/r/rust"; + let r = canonicalize_raw(raw).unwrap(); + assert_eq!(r.canonical, raw); + assert!(r.alias_of.is_none()); +} + +#[test] +fn parent_url_subreddit_under_r_hub() { + assert_eq!( + parent_url("https://reddit.com/r/movies").as_deref(), + Some("https://reddit.com/r") + ); +} + +#[test] +fn parent_url_domain_has_none() { + assert_eq!(parent_url("https://reddit.com").as_deref(), None); +} + +#[test] +fn parent_url_generic_site() { + assert_eq!( + parent_url("https://example.com/a/b").as_deref(), + Some("https://example.com/a") + ); +} + +#[test] +fn breadcrumbs_from_canonical_string_roundtrip() { + let id = "https://reddit.com/r/golang/comments/abc123"; + let crumbs = navigable_breadcrumbs(id); + assert_eq!(crumbs.last().map(String::as_str), Some(id)); +} + +// --- Table: Reddit raw URLs → canonical --- + +#[test] +fn reddit_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ("r/rust", "https://reddit.com/r/rust"), + ("/r/aww", "https://reddit.com/r/aww"), + ("https://reddit.com/r/rust", "https://reddit.com/r/rust"), + ( + "https://www.reddit.com/r/programming/new", + "https://reddit.com/r/programming", + ), + ( + "https://old.reddit.com/r/test/comments/xyz/slug/", + "https://reddit.com/r/test/comments/xyz", + ), + ( + "reddit.com/r/Movies/comments/abc/Title_Case_Slug", + "https://reddit.com/r/movies/comments/abc", + ), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Table: YouTube raw URLs → canonical --- + +#[test] +fn youtube_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ( + "https://youtube.com/watch?v=abc", + "https://youtube.com/watch?v=abc", + ), + ( + "https://www.youtube.com/watch?v=abc&t=1&feature=share", + "https://youtube.com/watch?v=abc", + ), + ("https://youtu.be/abc", "https://youtube.com/watch?v=abc"), + ( + "https://youtube.com/shorts/abc", + "https://youtube.com/watch?v=abc", + ), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Table: generic sites --- + +#[test] +fn generic_canonical_matrix() { + let cases: &[(&str, &str)] = &[ + ( + "https://news.ycombinator.com/item?id=38472", + "https://news.ycombinator.com/item?id=38472", + ), + ( + "https://www.github.com/rust-lang/rust/issues/1?utm_source=x", + "https://github.com/rust-lang/rust/issues/1", + ), + ("https://example.com", "https://example.com"), + ]; + for (raw, want) in cases { + assert_eq!(canon(raw), *want, "raw={raw}"); + } +} + +// --- Phantom /comments/ regression (sorter2-specific) --- + +#[test] +fn phantom_comments_not_in_breadcrumbs_for_post() { + let crumbs = navigable_breadcrumbs("https://reddit.com/r/rust/comments/aaa"); + assert!(!crumbs.iter().any(|c| c.ends_with("/comments"))); +} + +#[test] +fn phantom_comments_not_sibling_of_subreddit_in_breadcrumb_chain() { + let crumbs = navigable_breadcrumbs("https://reddit.com/r/rust/comments/aaa"); + let subs: Vec<_> = crumbs + .iter() + .filter(|c| c.contains("/r/rust") && !c.contains("/comments/")) + .collect(); + assert_eq!(subs, vec!["https://reddit.com/r/rust"]); +} + +// --- Distinct items must stay distinct --- + +#[test] +fn different_posts_different_canonical() { + let a = canon("https://reddit.com/r/rust/comments/aaa"); + let b = canon("https://reddit.com/r/rust/comments/bbb"); + assert_ne!(a, b); +} + +#[test] +fn different_subreddits_different_canonical() { + assert_ne!( + canon("https://reddit.com/r/rust"), + canon("https://reddit.com/r/golang") + ); +} Side B — contributor: tommy-mor Side B — commit message: [285b64d4] Offload Reddit payloads to RocksDB and stream event log replay. Vendor durable as a workspace crate, store entity JSON in entity_db instead of GlobalTree, and replay events.jsonl one line at a time to cut startup RAM. Co-authored-by: Cursor Side B — unified diff (full patch): diff --git a/Cargo.lock b/Cargo.lock index e55d87f32ab32064686431c7082ef8c9ca872d63..8fc09f9ac978bd7ccf57f177989057b606677db8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.102" @@ -56,6 +68,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "axum" version = "0.7.9" @@ -153,6 +171,75 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.2", + "shlex", + "syn", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -171,6 +258,22 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.62" @@ -178,15 +281,89 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cookie" version = "0.18.1" @@ -224,6 +401,73 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "deranged" version = "0.5.8" @@ -250,6 +494,25 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "durable" +version = "0.1.0" +dependencies = [ + "bincode", + "criterion", + "proptest", + "rocksdb", + "serde", + "tempfile", + "thiserror", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -373,6 +636,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -381,11 +656,17 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "h2" version = "0.4.14" @@ -405,6 +686,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -426,6 +718,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "http" version = "1.4.1" @@ -676,12 +974,51 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.99" @@ -700,6 +1037,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -712,6 +1055,43 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "librocksdb-sys" +version = "0.11.0+8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +dependencies = [ + "bindgen 0.65.1", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -730,6 +1110,16 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -781,6 +1171,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.2.0" @@ -826,6 +1222,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -841,19 +1247,34 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl" version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cfg-if", "foreign-types", "libc", @@ -890,6 +1311,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -908,6 +1335,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -974,6 +1429,31 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.11.1", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.45" @@ -983,6 +1463,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -996,8 +1482,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1007,7 +1503,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1019,6 +1525,56 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1090,13 +1646,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rocksdb" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -1142,12 +1720,33 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -1163,7 +1762,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -1307,9 +1906,10 @@ dependencies = [ "axum", "axum-extra", "dotenvy", + "durable", "futures-util", "maud", - "rand", + "rand 0.8.6", "reqwest", "serde", "serde_json", @@ -1378,7 +1978,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -1476,6 +2076,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.52.3" @@ -1558,7 +2168,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "http", "http-body", @@ -1575,7 +2185,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -1667,6 +2277,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1727,6 +2343,25 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1843,7 +2478,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap", "semver", @@ -1859,6 +2494,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -2040,7 +2684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.1", "indexmap", "log", "serde", @@ -2184,3 +2828,14 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "bindgen 0.72.1", + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 156820ec5151b709a254dafc177da10488fee566..89f57d8943de978ef6c2dc038a5dfb89420c4283 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["server"] +members = ["server", "durable"] resolver = "2" diff --git a/Dockerfile b/Dockerfile index 38e4d595f2366b6a6e9405f1e12373b70fac79a1..517aa49ea2c01ba0d06a4647259ed917a0f6665c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM rust:1.88-slim AS builder WORKDIR /build RUN apt-get update && \ - apt-get install -y pkg-config libssl-dev && \ + apt-get install -y pkg-config libssl-dev clang && \ rm -rf /var/lib/apt/lists/* COPY . . diff --git a/durable/.gitignore b/durable/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..90c273ef12d6313594dde541c3465f2a47739729 --- /dev/null +++ b/durable/.gitignore @@ -0,0 +1,35 @@ +all.txt + +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# Generated by cargo mutants +# Contains mutation testing data +**/mutants.out*/ + +# RustRover +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + + +# Added by cargo + +/target + + +# Added by cargo +# +# already existing elements were commented out + +#/target diff --git a/durable/Cargo.lock b/durable/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..99d59ec2166a1e88dc4867d316b3d5fe9bb0a1b6 --- /dev/null +++ b/durable/Cargo.lock @@ -0,0 +1,1198 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", +] + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "durable" +version = "0.1.0" +dependencies = [ + "bincode", + "criterion", + "proptest", + "rocksdb", + "serde", + "tempfile", + "thiserror", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets 0.53.2", +] + +[[package]] +name = "librocksdb-sys" +version = "0.11.0+8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +dependencies = [ + "bindgen 0.65.1", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.9.1", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rocksdb" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags 2.9.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.1", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "bindgen 0.71.1", + "cc", + "pkg-config", +] diff --git a/durable/Cargo.toml b/durable/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..907e79cd894446e2ac76c8240b86c2927103727b --- /dev/null +++ b/durable/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "durable" +version = "0.1.0" +edition = "2021" +authors = ["Durable Contributors"] +description = "RocksDB-backed persistent data structures for Rust" +license = "MIT OR Apache-2.0" + +[dependencies] +rocksdb = "0.21" +serde = { version = "1.0", features = ["derive"] } +bincode = "1.3" +thiserror = "1.0" + +[dev-dependencies] +tempfile = "3.8" +criterion = "0.5" +proptest = "1.4" diff --git a/durable/LICENSE b/durable/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/durable/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/durable/README.md b/durable/README.md new file mode 100644 index 0000000000000000000000000000000000000000..46f3b163a41d1094334ea5dc4e8add7b627f16e2 --- /dev/null +++ b/durable/README.md @@ -0,0 +1,207 @@ +# Durable + +RocksDB-backed persistent data structures for Rust. Think `std::collections` but on disk! + +## Features + +- **Persistent Collections**: `DurableVec`, `DurableMap`, `DurableSet` (coming soon) +- **Type-Safe**: Full Rust type safety with serde serialization +- **ACID Guarantees**: All operations are atomic and crash-safe +- **Zero-Copy Capable**: Efficient iteration without loading entire collections +- **Embedded**: No external services required - just a directory on disk + +## Quick Start + +Add to your `Cargo.toml`: + +```toml +[dependencies] +durable = "0.1.0" +``` + +## Example + +### DurableVec +```rust +use durable::{Db, DurableVec}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Serialize, Deserialize)] +struct Task { + id: u64, + title: String, + completed: bool, +} + +fn main() -> Result<(), Box> { + // Open or create a database + let db = Db::open("my_db")?; + + // Create a persistent vector + let mut tasks = DurableVec::::new(&db, "tasks")?; + + // Use it like a normal Vec! + tasks.push(Task { + id: 1, + title: "Build something amazing".to_string(), + completed: false, + })?; + + // Data persists across program restarts + println!("Total tasks: {}", tasks.len()?); + + Ok(()) +} +``` + +### DurableMap +```rust +use durable::{Db, DurableMap}; + +fn main() -> Result<(), Box> { + let db = Db::open("my_db")?; + + // Create a persistent map + let mut scores = DurableMap::::new(&db, "scores")?; + + // Use it like a HashMap! + // Use put() when you don't need the old value (more efficient) + scores.put("Alice".to_string(), 100)?; + scores.put("Bob".to_string(), 85)?; + + // Use insert() when you need to know the old value + if let Some(old_score) = scores.insert("Alice".to_string(), 120)? { + println!("Alice's previous score was: {}", old_score); + } + + // Get values + if let Some(score) = scores.get(&"Alice".to_string())? { + println!("Alice's score: {}", score); + } + + // Iterate over entries + for (name, score) in scores.iter()? { + println!("{}: {}", name, score); + } + + Ok(()) +} +``` + +### Nested Collections + +Durable supports nesting collections within each other for complex data structures: + +```rust +use durable::{Db, DurableMap, DurableVec}; + +fn main() -> Result<(), Box> { + let db = Db::open("my_db")?; + + // Create a map where each user has a list of posts + let user_posts: DurableMap> = + DurableMap::new_nested(&db, "user_posts"); + + // Add posts for a user + let mut alice_posts = user_posts.entry("alice".to_string())?.or_default()?; + alice_posts.push("Hello, world!".to_string())?; + alice_posts.push("Rust is awesome!".to_string())?; + + // Or use chained calls for convenience + user_posts.entry("bob".to_string())?.or_default()?.push("First post!".to_string())?; + + // Access nested data + let alice_posts = user_posts.entry("alice".to_string())?.or_default()?; + println!("Alice has {} posts", alice_posts.len()?); + + Ok(()) +} +``` + +The entry API automatically creates nested collections when they don't exist, providing ergonomic access patterns similar to `std::collections::HashMap::entry().or_default()`. + +## Current Status + +### Implemented + +- ✅ `DurableVec` with full test coverage including: + - Basic operations: `push`, `pop`, `get`, `len`, `clear` + - Batch operations: `extend` + - Iteration: `iter()` returns a streaming iterator, `to_vec()` loads into memory + - Property-based testing with proptest + - Unicode string support + - Complex type support + +- ✅ `DurableMap` with full test coverage including: + - Basic operations: `insert`, `put`, `get`, `remove`, `contains_key`, `len`, `clear` + - Batch operations: `extend` + - Iteration: `iter()`, `keys()`, `values()` return streaming iterators + - Memory loading: `to_vec()`, `keys_vec()`, `values_vec()` for convenience + - Complex key and value types + - Property-based testing with proptest + +- ✅ **Nested Collections** with entry API: + - `DurableMap>` - Maps to vectors + - `entry()` method with `or_default()` for ergonomic access + - Automatic collection creation and management + - Full persistence and isolation between nested collections + - Type-safe compile-time enforcement + +### Coming Soon + +- 🚧 `DurableSet` - Persistent HashSet +- 🚧 Deep nesting (e.g., `DurableMap>>`) +- 🚧 Schema migration support +- 🚧 Batch operations across multiple collections + +## Performance + +All operations are designed to be efficient: + +- **DurableVec**: + - `push`: Single atomic write with WAL flush + - `get`: Direct key lookup, O(1) + - `len`: Metadata lookup, O(1) + - `extend`: Batched writes for efficiency + - `clear`: Atomic batch deletion + +- **DurableMap**: + - `insert`: Returns old value (2 ops: get + put), O(1) average + - `put`: No return value (1 op: existence check + put), O(1) average + - `get`: Direct key lookup, O(1) average + - `remove`: Single delete with WAL flush + - `len`: Metadata lookup, O(1) + - `extend`: Batched writes for efficiency + +## Testing + +Run the test suite: + +```bash +cargo test +``` + +Run the examples: + +```bash +cargo run --example vec_example +cargo run --example map_example +cargo run --example combined_example # Shows both collections working together +cargo run --example streaming_demo # Demonstrates efficient streaming iteration +cargo run --example nested_example # Shows nested collections (Map -> Vec) +cargo run --example simple_ranking # Gaming leaderboard from docs/motivation.md +cargo run --example ranking_history # Complex ranking system with persistence +``` + +## License + +Licensed under either of: + +- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. diff --git a/durable/docs/001.md b/durable/docs/001.md new file mode 100644 index 0000000000000000000000000000000000000000..3ad7cf9d93572de9ded367f9ce6dcc28608bba39 --- /dev/null +++ b/durable/docs/001.md @@ -0,0 +1,272 @@ +## Durable — RocksDB-backed Persistent Data Structures for Rust + +### *draft RFC v0.1* + +--- + +### 1  Purpose + +Provide ergonomic, std-like collections (`DurableMap`, `DurableVec`, `DurableSet`, …) whose contents are **durably stored in RocksDB** yet feel in-memory: + +```rust +let db = durable::open("db"); // single call opens RocksDB +let mut posts = DurableMap::::new(&db, "posts")?; + +posts.insert(id, post)?; // ACID write +let p = posts.get(&id)?; // typed read +``` + +Target use cases: + +| Domain | Why durable? | +| ------------------------------ | -------------------------------- | +| Local-first apps / CRDT caches | crash-safe, embedded | +| High-write time-series blobs | append-only keys, fast iteration | +| ML / ranking histories | vector-within-vector patterns | +| Game state / async board games | snapshot + rollback | +| Sorter (tag → bucket → item) | exactly the motivating structure | + +--- + +### 2  Design Goals + +1. **Ergonomic** – Feel like `std::collections`; no manual key mangling. +2. **Typed** – Keys & values are generic; (de)serialization pluggable (default = `bincode`). +3. **Atomic** – Multi-op `batch.commit()` gives RocksDB write-batch semantics. +4. **Crash-safe** – Every public write path `fsync`s RocksDB WAL first. +5. **Composable** – Any collection may nest another via *prefix subspaces*. +6. **Zero external services** – Single embedded `.sst` directory. +7. **Opt-in features** – Reactive diffs, LRU cache, metrics behind feature flags. + +Non-goals (v0 line): + +* Distributed replication +* Multi-process concurrency (one writer process is assumed; readers may open secondary Rocks instances) +* SQL-style ad-hoc queries – you iterate, not query-plan. + +--- + +### 3  Key–Value Layout + +*Each collection owns a **prefix** inside a single Column Family (`default` by default).* + +``` + 0x00 0x00 [ 0x00 …] +``` + +* `user-prefix` = crate-level namespace (allows multi-tenant apps) +* `col_id` = 8-byte, little-endian numeric ID assigned on `DurableMap::new(&db,"posts")`. +* `logical-key` = `serde`-encoded key (or ordinal for Vec). +* `subindex` = extra path segments used by nested collections (e.g., `Vec` elements in a map value). + +Because RocksDB stores keys lexicographically, all elements of a collection (and its descendants) live contiguously → range scans & prefix deletes are cheap. + +--- + +### 4  Public API (surface) + +```rust +/// Opens (or creates) a durable database at `path`. +pub fn open>(path: P) -> Result; + +/// A transactional write batch +pub struct Batch<'db> { /* .. */ } + +impl<'db> Batch<'db> { + pub fn put(&mut self, col: &impl WriteCollection, key: &K, val: &V) -> Result<()>; + pub fn delete(&mut self, col: &impl WriteCollection, key: &K) -> Result<()>; + pub fn commit(self) -> Result<()>; +} + +/// Collections ------------------------------------------------------------ + +pub struct DurableMap<'db, K, V> { /* .. */ } +pub struct DurableVec<'db, T> { /* .. */ } +pub struct DurableSet<'db, T> { /* .. */ } +pub struct DurableIndex<'db, K, V> { /* sorted-map, range queries */ } + +/// Common read API +pub trait ReadCollection { + fn get(&self, k: &K) -> Result>; + fn contains_key(&self, k: &K) -> Result; + fn len(&self) -> Result; + fn iter(&self) -> Iter<'_, (K,V)>; // owns snapshot +} + +/// Common write API (auto batch or explicit) +pub trait WriteCollection: ReadCollection { + fn insert(&mut self, k: K, v: V) -> Result>; + fn remove(&mut self, k: &K) -> Result>; + fn clear(&mut self) -> Result<()>; +} +``` + +*All writes go through an internal `rocksdb::WriteBatch`* so a single logical op remains atomic even if it touches subkeys (e.g., pushing into a `DurableVec` updates `len` meta key + appends element). + +--- + +### 5  Collection Semantics + +#### 5.1 `DurableVec` + +* Meta-key `__len` stores current length (`u64`). +* Element key = `|` +* `push(elem)` = `batch.put(key(len), elem); batch.put(__len, len+1)` + – O(1) write, O(log N) read via prefix seek. +* `iter()` performs a prefix range; snapshot guarantees repeat-read. + +#### 5.2 `DurableMap` + +* Key = `|` +* `len` optional (feature `"size_tracking"`); otherwise O(prefix-scan). + +#### 5.3 Nested Collections + +```rust +let users = DurableMap::>::new(&db,"users")?; +users.entry("alice")?.or_default()?.push(order)?; +``` + +Internally `Entry::or_default` creates a *child prefix* off the parent’s key: +`|"alice"|0x00||…` + +Child collections store their own metadata keys beneath that path. + +--- + +### 6  Transactions & Consistency + +* **Auto-batch**: default mutator methods create a WriteBatch, commit, and flush WAL. +* **Explicit batch**: user opens `let mut wb = db.batch();`, issues puts/deletes via collection adapters, then `wb.commit()` for cross-collection atomicity. +* **Crash guarantee**: after `commit` returns, updates survive power loss (`rocksdb::DB::flush_wal(true)`). + +Read operations take a **consistent snapshot** by default; advanced users can opt-out for max throughput. + +--- + +### 7  Migrations (v0.2 roadmap) + +* Each collection stores a `u32 schema_version` meta key. +* `durable::open` accepts an optional `Schema` describing: + + ```rust + struct Schema { collections: Vec, version: u32 } + ``` + + If version mismatch ⇒ run user-supplied `migrate(old, new, &db)` which gets a mutable view and may batch-rewrite keys. + +--- + +### 8  Reactive Diffs (feature `"watch"`) + +* Behind a Tokio-aware feature; uses RocksDB’s `get_updates_since(seq)` API. +* `watch_prefix(prefix) -> impl Stream` + – `Diff` = key, old val (Option), new val (Option). +* Back-pressure handled with an in-process ring buffer; user chooses lag policy. + +--- + +### 9  Caching Layer (feature `"cache"`) + +* Probabilistic LRU over deserialized values. +* Configurable per-collection: `with_cache(cap_entries, ttl_ms)`. +* Coherent: write path invalidates cache keys on commit. + +--- + +### 10  Error Model + +```rust +#[non_exhaustive] +pub enum Error { + Rocks(rocksdb::Error), + Serde(bincode::Error), + Corruption(String), + TransactionAborted, + // feature-gated variants e.g. WatchLagged +} +``` + +`Result = std::result::Result` everywhere. + +--- + +### 11  Performance Budget (baseline targets) + +| Operation | Goal | +| -------------------- | -------------------------------------- | +| `map.insert` | < 30 µs (including WAL fsync) | +| `vec.push` | < 25 µs | +| Prefix iteration 1 M | > 120 MB/s read BW on NVMe | +| Concurrent readers | Linear scaling up to RocksDB read-IOPS | +| Watch latency | < 5 ms p50 on local SSD | + +(bench harness lives in `benches/` via Criterion.) + +--- + +### 12  Dependency Footprint + +* `rocksdb` (–> FB fork) ⟹ builds C++11 static lib (\~10 MB) +* `bincode` (default), with `serde` feature. +* `tokio-stream` only if `watch` feature enabled. + +MSRV = 1.76. + +--- + +### 13  Minimum Deliverable for **v0.1-alpha** + +* [ ] `Db::open`, `Db::batch` +* [ ] `DurableMap`, `DurableVec` +* [ ] automatic serialization via `serde` +* [ ] atomic commit + WAL flush +* [ ] snapshot reads +* [ ] unit tests (insert/get/iter/crash-recovery using `tempdir`) +* [ ] criterion bench + +--- + +### 14  Future Work + +* **Replicated mode** (Raft or FoundationDB layer) +* **CRDT merge semantics** for offline edits +* **DurableGraph** (adjacency lists + index) +* WebAssembly key-value adapters (edge workers) +* `tracing` instrumentation & Prometheus metrics + +--- + +### 15  Licensing & Governance + +* License: **Apache-2.0 OR MIT** (standard Rust dual license) +* Code-of-conduct: Rust CoC template +* Contribution model: PR + mandatory CI (fmt, clippy, test, bench) +* Early roadmap guided by original author(s); transfer to an org when ≥3 maintainers. + +--- + +## Appendix A — Sorter Use-Case Sketch + +```rust +type TagId = String; +type Bucket = u64; // logical Unix day or version # +type ItemId = String; +type Elo = i32; + +let hist = DurableMap::>>::new(&db,"hist")?; + +// update elo +hist.entry("tf2")? + .or_default()? + .entry(today_bucket)? + .or_default()? + .push((item_id, new_elo))?; + +// stream bucket +let items = hist.get("tf2")?.unwrap() + .get(&today_bucket)?.unwrap() + .iter().collect::>(); +``` + +All three layers share one RocksDB instance; you pay one WAL flush per ranking update, but reads are prefix-scans with zero allocations. diff --git a/durable/docs/motivation.md b/durable/docs/motivation.md new file mode 100644 index 0000000000000000000000000000000000000000..50d8ee5ef9f384d908e3a86f6945e3f4ced58292 --- /dev/null +++ b/durable/docs/motivation.md @@ -0,0 +1,248 @@ +# Why Durable? The Missing Abstraction Layer for Persistent Storage + +## The Problem: The Abstraction Gap + +Every database forces developers to translate between how they **think** about data and how they **store** it. This translation layer is where bugs hide, performance suffers, and development slows down. + +### Example: Building a Multiplayer Game Leaderboard + +Let's say you need to store player rankings by game mode, with history by day. Here's the data model in your head: + +``` +Game Mode → Day → List of (Player, Score) +``` + +#### With Raw Key-Value Stores (sled, RocksDB) + +```rust +// Storing a score requires manual key construction +let key = format!("leaderboard:{}:{}:player:{}", game_mode, day, player_id); +db.insert(key.as_bytes(), score.to_le_bytes())?; + +// Getting today's leaderboard? Manual prefix scan and deserialization +let prefix = format!("leaderboard:{}:{}:", game_mode, day); +let mut scores = Vec::new(); +for item in db.scan_prefix(prefix.as_bytes()) { + let (key, value) = item?; + // Parse player_id from key string... hope the format is right + // Deserialize score... hope it's the right type + scores.push((player_id, score)); +} +scores.sort_by_key(|(_, s)| *s); + +// Want to know how many players played today? Another scan! +// Want to atomically update multiple scores? Write a transaction wrapper! +// Want to clean up old days? Manual prefix iteration and deletion! +``` + +**Problems:** +- String manipulation for every operation +- No type safety (everything is bytes) +- Manual implementation of collection semantics +- No atomicity across related keys +- Performance overhead from string parsing + +#### With SQL Databases (SQLite, PostgreSQL) + +```sql +CREATE TABLE leaderboards ( + game_mode VARCHAR(50), + day DATE, + player_id UUID, + score INTEGER, + PRIMARY KEY (game_mode, day, player_id) +); + +-- Getting a leaderboard requires SQL +SELECT player_id, score +FROM leaderboards +WHERE game_mode = ? AND day = ? +ORDER BY score DESC; +``` + +```rust +// In Rust, you need an ORM or manual query building +let scores: Vec<(Uuid, i32)> = sqlx::query_as( + "SELECT player_id, score FROM leaderboards WHERE game_mode = $1 AND day = $2 ORDER BY score DESC" +) +.bind(&game_mode) +.bind(&day) +.fetch_all(&pool) +.await?; +``` + +**Problems:** +- Impedance mismatch (relational model vs nested structures) +- SQL complexity for simple operations +- ORMs add abstraction layers and performance overhead +- Async runtime required even for local storage +- Schema migrations for every structural change + +#### With Document Stores (MongoDB) + +```javascript +// Document structure +{ + game_mode: "ranked", + day: "2024-01-15", + scores: [ + { player_id: "abc", score: 1500 }, + { player_id: "def", score: 1400 } + ] +} + +// But now you have a different problem: updating a single score +// requires loading and saving the entire document! +``` + +**Problems:** +- Not embedded (requires separate process) +- Document size limits +- Inefficient for partial updates +- Complex setup for local-first apps + +## The Solution: Native Data Structures + +With Durable, you express your data model directly: + +```rust +let leaderboard = DurableMap::>>::new(&db, "leaderboard")?; + +// Store a score - reads like natural Rust code +leaderboard + .entry(game_mode)? + .or_default()? + .entry(day)? + .or_default()? + .push((player_id, score))?; + +// Get today's leaderboard - it's just a Vec +let mut today_scores = leaderboard + .get(&game_mode)? + .and_then(|mode| mode.get(&day).ok()) + .unwrap_or_default(); +today_scores.sort_by_key(|(_, s)| *s); + +// All operations are atomic, typed, and efficient +``` + +## Why This Matters + +### 1. **Zero Translation Overhead** + +Your mental model **is** the storage model. No more: +- String concatenation for keys +- Manual serialization/deserialization +- SQL query construction +- Document structure mapping + +### 2. **Composition Without Complexity** + +Nested data structures "just work": + +```rust +// A real-world example: user notifications by app by priority +let notifications = DurableMap::>>>::new(&db, "notifs")?; + +// Natural access patterns +notifications + .get(&user_id)? + .get(&app_id)? + .get(&Priority::High)? + .iter() + .take(10) // Latest 10 high-priority notifications +``` + +Try implementing this with SQL joins or KV prefixes! + +### 3. **Type Safety Throughout** + +```rust +// This won't compile - type safety at every level +let score: String = leaderboard.get(&"chess")?.get(&20240115)?.get(0)?; +// ^^^^^^ expected Score, found String + +// With raw KV stores, this is a runtime error after deserialization +``` + +### 4. **Atomicity By Design** + +```rust +// Multiple operations in one atomic batch +let mut batch = db.batch(); +batch.vec_push(&game.players, new_player)?; +batch.map_insert(&game.scores, player_id, 0)?; +batch.map_increment(&game.stats, "player_count", 1)?; +batch.commit()?; // All or nothing +``` + +### 5. **Performance Without Compromise** + +- **Locality**: Related data stored contiguously (prefix design) +- **Zero-copy possible**: Direct memory mapping for read-heavy workloads +- **Streaming iteration**: No need to load entire collections +- **Bulk operations**: Native batch support + +## Comparison Matrix + +| Feature | Durable | sled/RocksDB | SQLite | MongoDB | +|---------|---------|--------------|---------|----------| +| **Native collections** | ✅ Built-in | ❌ DIY | ❌ Tables only | ⚠️ Documents | +| **Type safety** | ✅ Full | ❌ Bytes | ⚠️ ORM-dependent | ⚠️ Schema validation | +| **Nested structures** | ✅ Natural | ❌ Manual prefixes | ❌ Joins/JSON | ✅ Embedded docs | +| **Atomic operations** | ✅ Automatic | ⚠️ Manual batching | ✅ Transactions | ⚠️ Document-level | +| **Local/embedded** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ Separate process | +| **Schema evolution** | ✅ Per-collection | ❌ DIY | ⚠️ Migrations | ✅ Flexible | +| **Memory efficiency** | ✅ Scan & stream | ✅ Manual | ⚠️ Query-dependent | ❌ Doc loading | + +## Real-World Use Cases Where Durable Shines + +### Local-First Sync Engine + +```rust +// Sync state with conflict tracking +let sync_state = DurableMap::>::new(&db, "sync")?; + +// Natural conflict detection +let versions = sync_state.get(&record_id)?; +if versions.values().unique().count() > 1 { + // Conflict detected - handle naturally +} +``` + +### Time-Series Analytics Cache + +```rust +// Metrics by source by minute +let metrics = DurableMap::>>::new(&db, "metrics")?; + +// Natural windowing +let last_hour: Vec = metrics + .get(&source)? + .range(now - 3600..=now)? + .flat_map(|(_, minute_metrics)| minute_metrics.iter()) + .collect(); +``` + +### Feature Flag System with History + +```rust +// Flags by environment with change history +let flags = DurableMap::>>::new(&db, "flags")?; + +// Natural audit trail +let history = flags.get(&Env::Prod)?.get("new-feature")?.iter().collect(); +``` + +## The Bottom Line + +**Durable isn't a better database - it's the missing abstraction layer that lets you use persistent storage like in-memory collections.** + +Stop translating. Start building. + +--- + +*Next: Read the [RFC](001.md) for implementation details, or jump to the [Quick Start Guide](quickstart.md).* \ No newline at end of file diff --git a/durable/examples/combined_example.rs b/durable/examples/combined_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..626a6e1cf7e3c9c26f9f2edc58950d9bc31ec67e --- /dev/null +++ b/durable/examples/combined_example.rs @@ -0,0 +1,160 @@ +use durable::{Db, DurableMap, DurableVec}; +use serde::{Serialize, Deserialize}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Message { + id: u64, + from: String, + to: String, + content: String, + timestamp: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct User { + username: String, + display_name: String, + message_count: u32, +} + +fn get_timestamp() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +fn main() -> Result<(), Box> { + // Open or create a database + let db = Db::open("chat_db")?; + + // Create our collections + let mut users = DurableMap::::new(&db, "users")?; + let mut messages = DurableVec::::new(&db, "messages")?; + let mut user_message_indices = DurableMap::>::new(&db, "user_messages")?; + + // Create some users + users.insert("alice".to_string(), User { + username: "alice".to_string(), + display_name: "Alice Smith".to_string(), + message_count: 0, + })?; + + users.insert("bob".to_string(), User { + username: "bob".to_string(), + display_name: "Bob Johnson".to_string(), + message_count: 0, + })?; + + users.insert("charlie".to_string(), User { + username: "charlie".to_string(), + display_name: "Charlie Brown".to_string(), + message_count: 0, + })?; + + // Helper to send a message + let send_message = |from: &str, to: &str, content: &str, + messages: &mut DurableVec, + users: &mut DurableMap, + indices: &mut DurableMap>| -> Result<(), Box> { + // Create message + let msg_id = messages.len()? as u64; + let message = Message { + id: msg_id, + from: from.to_string(), + to: to.to_string(), + content: content.to_string(), + timestamp: get_timestamp(), + }; + + // Store message + messages.push(message)?; + let msg_index = messages.len()? - 1; + + // Update sender's message count + if let Some(mut sender) = users.get(&from.to_string())? { + sender.message_count += 1; + users.insert(from.to_string(), sender)?; + } + + // Track message indices for recipient + let mut recipient_indices = indices.get(&to.to_string())?.unwrap_or_default(); + recipient_indices.push(msg_index); + indices.insert(to.to_string(), recipient_indices)?; + + Ok(()) + }; + + // Send some messages + println!("💬 Chat Application Demo\n"); + println!("Sending messages..."); + + send_message("alice", "bob", "Hey Bob, how's the Durable library coming along?", + &mut messages, &mut users, &mut user_message_indices)?; + + send_message("bob", "alice", "It's going great! We have DurableVec and DurableMap working!", + &mut messages, &mut users, &mut user_message_indices)?; + + send_message("charlie", "alice", "That sounds awesome! Can I help with testing?", + &mut messages, &mut users, &mut user_message_indices)?; + + send_message("alice", "charlie", "Absolutely! The more testing the better!", + &mut messages, &mut users, &mut user_message_indices)?; + + send_message("bob", "charlie", "Check out the examples directory for usage patterns", + &mut messages, &mut users, &mut user_message_indices)?; + + // Display all users and their message counts + println!("\n👥 Users:"); + let mut all_users = users.to_vec()?; + all_users.sort_by_key(|(username, _)| username.clone()); + + for (username, user) in all_users { + println!(" {} ({}) - {} messages sent", + user.display_name, username, user.message_count); + } + + // Display all messages + println!("\n📨 All messages:"); + for (i, msg) in messages.iter()?.enumerate() { + let msg = msg?; + println!(" [{}] {} → {}: {}", i, msg.from, msg.to, msg.content); + } + + // Show inbox for each user + println!("\n📥 User inboxes:"); + for item in users.iter() { + let (username, _) = item?; + if let Some(indices) = user_message_indices.get(&username)? { + println!("\n {}'s inbox ({} messages):", username, indices.len()); + for &idx in &indices { + if let Some(msg) = messages.get(idx)? { + println!(" From {}: {}", msg.from, msg.content); + } + } + } + } + + // Statistics + println!("\n📊 Statistics:"); + println!(" Total users: {}", users.len()?); + println!(" Total messages: {}", messages.len()?); + + // Demonstrate persistence + println!("\n💾 Data has been persisted to disk!"); + println!(" Database location: ./chat_db"); + + // Clean up + drop(messages); + drop(users); + drop(user_message_indices); + drop(db); + + // Remove the database for this example + std::fs::remove_dir_all("chat_db").ok(); + + println!("\n✅ Example completed!"); + + Ok(()) +} \ No newline at end of file diff --git a/durable/examples/map_example.rs b/durable/examples/map_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..08b8f2c8826caf53c4c20b422a540c92f6624029 --- /dev/null +++ b/durable/examples/map_example.rs @@ -0,0 +1,99 @@ +use durable::{Db, DurableMap}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct UserProfile { + name: String, + email: String, + score: u32, +} + +fn main() -> Result<(), Box> { + // Open or create a database + let db = Db::open("example_db")?; + + // Create a persistent map of user profiles + let mut users = DurableMap::::new(&db, "users")?; + + // Insert some users + // Using put() when we don't need the old value - more efficient! + users.put( + "alice".to_string(), + UserProfile { + name: "Alice Smith".to_string(), + email: "alice@example.com".to_string(), + score: 1500, + }, + )?; + + users.put( + "bob".to_string(), + UserProfile { + name: "Bob Johnson".to_string(), + email: "bob@example.com".to_string(), + score: 1200, + }, + )?; + + // Using insert() when we might need the old value + let old_charlie = users.insert( + "charlie".to_string(), + UserProfile { + name: "Charlie Brown".to_string(), + email: "charlie@example.com".to_string(), + score: 1800, + }, + )?; + + if old_charlie.is_some() { + println!("Replaced existing charlie entry"); + } + + println!("Total users: {}", users.len()?); + + // Look up a specific user + if let Some(alice) = users.get(&"alice".to_string())? { + println!("\nAlice's profile: {:?}", alice); + } + + // Check if a user exists + println!("\nDoes 'david' exist? {}", users.contains_key(&"david".to_string())?); + + // Update a user's score + if let Some(mut bob) = users.get(&"bob".to_string())? { + bob.score += 100; + // Use put() here since we don't need the old value back + users.put("bob".to_string(), bob)?; + println!("Updated Bob's score!"); + } + + // Iterate over all users + println!("\nAll users (sorted by username):"); + let mut all_users = users.to_vec()?; + all_users.sort_by_key(|(username, _)| username.clone()); + + for (username, profile) in all_users { + println!(" {} ({}) - Score: {}", username, profile.email, profile.score); + } + + // Get just the usernames + let mut usernames = users.keys_vec()?; + usernames.sort(); + println!("\nAll usernames: {:?}", usernames); + + // Find the highest scoring user + let profiles = users.values_vec()?; + if let Some(top_user) = profiles.iter().max_by_key(|p| p.score) { + println!("\nTop scorer: {} with {} points", top_user.name, top_user.score); + } + + // Remove a user + if let Some(removed) = users.remove(&"charlie".to_string())? { + println!("\nRemoved user: {}", removed.name); + println!("Users remaining: {}", users.len()?); + } + + println!("\nData has been persisted to disk."); + + Ok(()) +} \ No newline at end of file diff --git a/durable/examples/nested_example.rs b/durable/examples/nested_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..3b880f2c8b8ad2633d4b5fcf2016652216c9de8e --- /dev/null +++ b/durable/examples/nested_example.rs @@ -0,0 +1,64 @@ +use durable::{Db, DurableMap, DurableVec}; + +fn main() -> Result<(), Box> { + // Open a database + let db = Db::open("nested_example_db")?; + + // Create a map where each user has a list of posts + let user_posts: DurableMap> = DurableMap::new_nested(&db, "user_posts"); + + // Add posts for Alice + println!("Adding posts for Alice..."); + let mut alice_posts = user_posts.entry("alice".to_string())?.or_default()?; + alice_posts.push("Hello, world!".to_string())?; + alice_posts.push("Rust is awesome!".to_string())?; + alice_posts.push("Loving persistent data structures!".to_string())?; + + // Add posts for Bob + println!("Adding posts for Bob..."); + let mut bob_posts = user_posts.entry("bob".to_string())?.or_default()?; + bob_posts.push("First post".to_string())?; + bob_posts.push("Learning Rust".to_string())?; + + // Add a post for Charlie in a chained call + println!("Adding post for Charlie..."); + user_posts.entry("charlie".to_string())?.or_default()?.push("One-liner post!".to_string())?; + + // Read back Alice's posts + println!("\nAlice's posts:"); + let alice_posts_read = user_posts.entry("alice".to_string())?.or_default()?; + for i in 0..alice_posts_read.len()? { + if let Some(post) = alice_posts_read.get(i)? { + println!(" {}: {}", i + 1, post); + } + } + + // Read back Bob's posts + println!("\nBob's posts:"); + let bob_posts_read = user_posts.entry("bob".to_string())?.or_default()?; + for i in 0..bob_posts_read.len()? { + if let Some(post) = bob_posts_read.get(i)? { + println!(" {}: {}", i + 1, post); + } + } + + // Read back Charlie's posts + println!("\nCharlie's posts:"); + let charlie_posts_read = user_posts.entry("charlie".to_string())?.or_default()?; + for i in 0..charlie_posts_read.len()? { + if let Some(post) = charlie_posts_read.get(i)? { + println!(" {}: {}", i + 1, post); + } + } + + println!("\nDemonstration of persistence..."); + println!("Data is now persisted to disk. You can stop and restart this program,"); + println!("and all the posts will still be there!"); + + println!("\nTotal users with posts: 3"); + println!("Alice has {} posts", alice_posts_read.len()?); + println!("Bob has {} posts", bob_posts_read.len()?); + println!("Charlie has {} posts", charlie_posts_read.len()?); + + Ok(()) +} \ No newline at end of file diff --git a/durable/examples/ranking_history.rs b/durable/examples/ranking_history.rs new file mode 100644 index 0000000000000000000000000000000000000000..4da8592d240318318ae20c834d616210edb16c8d --- /dev/null +++ b/durable/examples/ranking_history.rs @@ -0,0 +1,179 @@ +use durable::{Db, DurableMap, DurableVec}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd)] +struct RankingEntry { + player_id: String, + score: i32, + timestamp: u64, +} + +impl RankingEntry { + fn new(player_id: &str, score: i32) -> Self { + Self { + player_id: player_id.to_string(), + score, + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } +} + +fn main() -> Result<(), Box> { + println!("🎮 Gaming Ranking History System (Rewritten with Nested Entry API)"); + println!("================================================================"); + + let db = Db::open("ranking_history_db")?; + + // THE CORE CHANGE: Define the truly nested data structure. + // Instead of a composite key, we nest a Map within a Map. + // This represents the ideal, ergonomic API. + type DailyRankings = DurableVec; + type GameHistory = DurableMap; + type Rankings = DurableMap; + + let rankings: Rankings = DurableMap::new_nested(&db, "game_rankings_v2"); + + // Simulate some game days + let today = 20241215u32; + let yesterday = 20241214u32; + let last_week = 20241208u32; + + // No more `make_key` helper function! + + println!("\n📊 Adding ranking data using chained entry().or_default()..."); + + // Add rankings for CS2 today. This demonstrates the new, clean access pattern. + println!("Adding CS2 rankings for today ({})", today); + let mut cs2_today = rankings + .entry("cs2".to_string())? + .or_default()? // Returns GameHistory (DurableMap) for "cs2" + .entry(today)? + .or_default()?; // Returns DailyRankings (DurableVec<...>) for `today` + + cs2_today.push(RankingEntry::new("player1", 2450))?; + cs2_today.push(RankingEntry::new("player2", 2380))?; + cs2_today.push(RankingEntry::new("player3", 2320))?; + cs2_today.push(RankingEntry::new("player4", 2280))?; + + // Add rankings for CS2 yesterday + println!("Adding CS2 rankings for yesterday ({})", yesterday); + rankings + .entry("cs2".to_string())? + .or_default()? + .entry(yesterday)? + .or_default()? + .push(RankingEntry::new("player1", 2420))?; + rankings + .entry("cs2".to_string())? + .or_default()? + .entry(yesterday)? + .or_default()? + .push(RankingEntry::new("player2", 2350))?; + rankings + .entry("cs2".to_string())? + .or_default()? + .entry(yesterday)? + .or_default()? + .push(RankingEntry::new("player5", 2300))?; + + // Add rankings for Valorant today + println!("Adding Valorant rankings for today ({})", today); + let mut valorant_today = rankings + .entry("valorant".to_string())? + .or_default()? + .entry(today)? + .or_default()?; + + valorant_today.push(RankingEntry::new("player6", 1850))?; + valorant_today.push(RankingEntry::new("player7", 1820))?; + valorant_today.push(RankingEntry::new("player1", 1800))?; // Same player, different game + + // Add TF2 rankings (matching the docs example) + println!("Adding TF2 rankings for last week ({})", last_week); + rankings + .entry("tf2".to_string())? + .or_default()? + .entry(last_week)? + .or_default()? + .push(RankingEntry::new("veteran_player", 3200))?; + rankings + .entry("tf2".to_string())? + .or_default()? + .entry(last_week)? + .or_default()? + .push(RankingEntry::new("old_school_gamer", 3150))?; + + println!("\n🏆 Reading back ranking data with the same natural API..."); + + // Get today's CS2 leaderboard + println!("\n🎯 CS2 Leaderboard for {} (today):", today); + let mut today_rankings = rankings + .entry("cs2".to_string())? + .or_default()? + .entry(today)? + .or_default()? + .to_vec()?; + + // Sort by score descending + today_rankings.sort_by(|a, b| b.score.cmp(&a.score)); + + for (rank, entry) in today_rankings.iter().enumerate() { + println!(" {}. {} - {} points", rank + 1, entry.player_id, entry.score); + } + + // Show cross-game analysis is still easy + println!("\n🎮 Multi-game player analysis for player1 on {}:", today); + let cs2_player1_score = today_rankings.iter() + .find(|e| e.player_id == "player1") + .map(|e| e.score); + + let valorant_player1_score = valorant_today.to_vec()?.iter() + .find(|e| e.player_id == "player1") + .map(|e| e.score); + + if let Some(score) = cs2_player1_score { println!(" CS2 Score: {}", score); } + if let Some(score) = valorant_player1_score { println!(" Valorant Score: {}", score); } + + // Showcase the power of the nested structure for stats + // For nested collections, we use the keys API instead of iter() + println!("\n📊 Dynamic Database Statistics (discovered games):"); + + // Note: For nested collections, we iterate over known keys or use a different approach + // since the values (nested DurableMaps) cannot be directly deserialized + let games = vec!["cs2", "valorant", "tf2"]; // In a real app, you might track these separately + + for game in games { + let game_history = rankings.entry(game.to_string())?.or_default()?; + let active_days = game_history.len()?; + + if active_days > 0 { + // For demonstration, let's count entries from known days + let mut total_entries = 0; + let days = [today, yesterday, last_week]; + + for day in days { + if let Ok(daily_rankings) = game_history.entry(day) { + if let Ok(rankings_vec) = daily_rankings.or_default() { + total_entries += rankings_vec.len()?; + } + } + } + + if total_entries > 0 { + println!(" • {}: {} total entries across {} active day(s)", + game.to_uppercase(), total_entries, active_days); + } + } + } + + println!("\n✨ Key Benefits of This Rewritten Approach:"); + println!(" • No more manual key construction (`format!`) - the core goal is met!"); + println!(" • The code's structure now mirrors the mental model: `rankings[game][day]`"); + println!(" • Truly compositional API, unlocking more powerful dynamic queries (like the stats section)"); + println!(" • Demonstrates the full power of the `DurableCollection` and `entry()` design."); + + Ok(()) +} \ No newline at end of file diff --git a/durable/examples/simple_ranking.rs b/durable/examples/simple_ranking.rs new file mode 100644 index 0000000000000000000000000000000000000000..3cc873c41f3d93c7fa9e6e2dfc80e815f456b1f9 --- /dev/null +++ b/durable/examples/simple_ranking.rs @@ -0,0 +1,68 @@ +use durable::{Db, DurableMap, DurableVec}; + +fn main() -> Result<(), Box> { + println!("🏆 Simple Game Ranking Example"); + println!("Demonstrating the pattern from docs/motivation.md"); + println!("==============================================="); + + let db = Db::open("simple_ranking_db")?; + + // This is the exact pattern from the docs: Game Mode → List of (Player, Score) + // For simplicity, we're showing one day's data per game mode + let rankings: DurableMap> = DurableMap::new_nested(&db, "rankings"); + + println!("\n📊 Adding TF2 rankings (from the docs example)..."); + + // This is the exact code pattern shown in docs/motivation.md + let mut tf2_rankings = rankings.entry("tf2".to_string())?.or_default()?; + tf2_rankings.push(("player1".to_string(), 1500))?; + tf2_rankings.push(("player2".to_string(), 1400))?; + tf2_rankings.push(("player3".to_string(), 1300))?; + + println!("✅ Added TF2 rankings using the docs pattern!"); + + // Add some other games for comparison + println!("\n📊 Adding CS2 rankings..."); + let mut cs2_rankings = rankings.entry("cs2".to_string())?.or_default()?; + cs2_rankings.push(("pro_player".to_string(), 2500))?; + cs2_rankings.push(("skilled_gamer".to_string(), 2200))?; + + println!("✅ Added CS2 rankings!"); + + // Now read back the data + println!("\n🏆 Current TF2 Leaderboard:"); + let tf2_data = rankings.entry("tf2".to_string())?.or_default()?; + + // Convert to vec and sort for display + let mut tf2_leaderboard = tf2_data.to_vec()?; + tf2_leaderboard.sort_by(|a, b| b.1.cmp(&a.1)); // Sort by score descending + + for (rank, (player, score)) in tf2_leaderboard.iter().enumerate() { + println!(" {}. {} - {} points", rank + 1, player, score); + } + + println!("\n🏆 Current CS2 Leaderboard:"); + let cs2_data = rankings.entry("cs2".to_string())?.or_default()?; + + let mut cs2_leaderboard = cs2_data.to_vec()?; + cs2_leaderboard.sort_by(|a, b| b.1.cmp(&a.1)); + + for (rank, (player, score)) in cs2_leaderboard.iter().enumerate() { + println!(" {}. {} - {} points", rank + 1, player, score); + } + + println!("\n📈 Database Statistics:"); + println!(" TF2 has {} players", tf2_data.len()?); + println!(" CS2 has {} players", cs2_data.len()?); + + println!("\n✨ This demonstrates the exact pattern from docs/motivation.md:"); + println!(" rankings.entry(game_mode)?.or_default()?.push((player, score))?;"); + println!(" "); + println!(" Compare this to the manual key construction required with raw KV stores:"); + println!(" let key = format!(\"leaderboard:{{}}:{{}}:player:{{}}\", game_mode, day, player_id);"); + println!(" db.insert(key.as_bytes(), score.to_le_bytes())?;"); + println!(" "); + println!(" Durable provides the ergonomic, type-safe abstraction over RocksDB!"); + + Ok(()) +} \ No newline at end of file diff --git a/durable/examples/streaming_demo.rs b/durable/examples/streaming_demo.rs new file mode 100644 index 0000000000000000000000000000000000000000..3a74a5318675f94f89aaf01562a5b28f8a1b664a --- /dev/null +++ b/durable/examples/streaming_demo.rs @@ -0,0 +1,81 @@ +use durable::{Db, DurableMap, DurableVec}; + +fn main() -> Result<(), Box> { + let db = Db::open("streaming_demo_db")?; + + // Create collections with a moderate amount of data + let mut map = DurableMap::::new(&db, "large_map")?; + let mut vec = DurableVec::::new(&db, "large_vec")?; + + println!("🚀 Streaming Iterator Demo\n"); + + // Add 1000 entries to demonstrate streaming + println!("Adding 1000 entries to map and vec..."); + for i in 0..1000 { + map.insert(i, format!("Value {}", i))?; + vec.push(format!("Item {}", i))?; + } + + println!("\n📊 Collection sizes:"); + println!(" Map entries: {}", map.len()?); + println!(" Vec elements: {}", vec.len()?); + + // Demonstrate streaming iteration - memory efficient + println!("\n✨ Streaming iteration (memory efficient):"); + + // Count items without loading into memory + let map_count = map.iter().count(); + println!(" Counted {} map entries without loading into memory", map_count); + + // Find specific items efficiently + let target = 500; + let found = map.iter() + .find(|item| { + item.as_ref() + .map(|(k, _)| *k == target) + .unwrap_or(false) + }); + + if let Some(Ok((k, v))) = found { + println!(" Found key {} with value '{}' via streaming", k, v); + } + + // Process only what we need + println!("\n🎯 Processing first 10 items only:"); + for (i, item) in vec.iter()?.take(10).enumerate() { + match item { + Ok(value) => println!(" [{}] {}", i, value), + Err(e) => println!(" [{}] Error: {:?}", i, e), + } + } + + // Filter and process without loading all data + println!("\n🔍 Filtering even keys without loading all data:"); + let even_count = map.keys() + .filter(|item| { + item.as_ref() + .map(|k| k % 2 == 0) + .unwrap_or(false) + }) + .count(); + println!(" Found {} even keys", even_count); + + // Compare with loading everything into memory + println!("\n⚠️ Loading all data into memory (less efficient for large collections):"); + let all_values = map.values_vec()?; + println!(" Loaded {} values into a Vec", all_values.len()); + + println!("\n✅ Streaming iterators provide:"); + println!(" • Constant memory usage regardless of collection size"); + println!(" • Ability to process data larger than RAM"); + println!(" • Early termination when finding specific items"); + println!(" • Efficient filtering and transformation"); + + // Clean up + drop(map); + drop(vec); + drop(db); + std::fs::remove_dir_all("streaming_demo_db").ok(); + + Ok(()) +} \ No newline at end of file diff --git a/durable/examples/vec_example.rs b/durable/examples/vec_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..07d394b5f3964ea1942a645c2f008e9e3c37d6d8 --- /dev/null +++ b/durable/examples/vec_example.rs @@ -0,0 +1,66 @@ +use durable::{Db, DurableVec}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct Task { + id: u64, + title: String, + completed: bool, +} + +fn main() -> Result<(), Box> { + // Open or create a database + let db = Db::open("example_db")?; + + // Create a persistent vector of tasks + let mut tasks = DurableVec::::new(&db, "tasks")?; + + // Add some tasks + tasks.push(Task { + id: 1, + title: "Build Durable library".to_string(), + completed: true, + })?; + + tasks.push(Task { + id: 2, + title: "Write comprehensive tests".to_string(), + completed: true, + })?; + + tasks.push(Task { + id: 3, + title: "Create documentation".to_string(), + completed: false, + })?; + + println!("Total tasks: {}", tasks.len()?); + + // Iterate through all tasks + println!("\nAll tasks:"); + for (i, task) in tasks.iter()?.enumerate() { + let task = task?; + println!(" [{}] {} - {}", + i, + task.title, + if task.completed { "✓" } else { "○" } + ); + } + + // Get a specific task + if let Some(task) = tasks.get(1)? { + println!("\nTask at index 1: {:?}", task); + } + + // Mark the last task as completed + if let Some(mut last_task) = tasks.pop()? { + println!("\nCompleting task: {}", last_task.title); + last_task.completed = true; + tasks.push(last_task)?; + } + + // The data persists even after the program exits! + println!("\nData has been persisted to disk."); + + Ok(()) +} \ No newline at end of file diff --git a/durable/repomix.config.json b/durable/repomix.config.json new file mode 100644 index 0000000000000000000000000000000000000000..bb52e4fcd383e8937245bd62566db8b7652402c2 --- /dev/null +++ b/durable/repomix.config.json @@ -0,0 +1,27 @@ +{ + "output": { + "filePath": "all.txt", + "style": "plain", + "parsableStyle": false, + "fileSummary": true, + "directoryStructure": true, + "removeComments": false, + "removeEmptyLines": false, + "compress": false, + "topFilesLength": 100, + "showLineNumbers": false, + "copyToClipboard": false + }, + "include": [], + "ignore": { + "useGitignore": true, + "useDefaultPatterns": true, + "customPatterns": [] + }, + "security": { + "enableSecurityCheck": true + }, + "tokenCount": { + "encoding": "o200k_base" + } +} diff --git a/durable/src/lib.rs b/durable/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..94e8823be93be1176b0a4cdc9797181b9f8caf60 --- /dev/null +++ b/durable/src/lib.rs @@ -0,0 +1,123 @@ +//! Durable - RocksDB-backed persistent data structures for Rust + +use std::path::Path; +use std::sync::Arc; +use rocksdb::{DB as RocksDB, Options, WriteBatch}; +use thiserror::Error; + +pub mod vec; +pub mod map; +pub use vec::DurableVec; +pub use map::DurableMap; + +/// Error types for Durable operations +#[derive(Error, Debug)] +pub enum DurableError { + #[error("RocksDB error: {0}")] + RocksDB(#[from] rocksdb::Error), + + #[error("Serialization error: {0}")] + Serialization(#[from] bincode::Error), + + #[error("Key not found")] + KeyNotFound, + + #[error("Collection not found: {0}")] + CollectionNotFound(String), + + #[error("Data corruption: {0}")] + Corruption(String), +} + +pub type Result = std::result::Result; + +/// A trait for types that can be used as nested collections. +pub trait DurableCollection { + /// Creates a new instance of the collection from a database handle + /// and a pre-determined, unique key prefix. + /// + /// This is the key method that allows `DurableMap` to instantiate + /// a nested collection handle. + fn from_prefix(db: Db, prefix: Vec) -> Self; +} + +/// The main database handle +#[derive(Clone)] +pub struct Db { + inner: Arc, +} + +impl Db { + /// Opens or creates a durable database at the given path + pub fn open>(path: P) -> Result { + let mut opts = Options::default(); + opts.create_if_missing(true); + opts.create_missing_column_families(true); + + let db = RocksDB::open(&opts, path)?; + Ok(Db { + inner: Arc::new(db), + }) + } + + /// Create a new write batch for atomic operations + pub fn batch(&self) -> Batch { + Batch { + db: self.clone(), + inner: WriteBatch::default(), + } + } + + /// Get the underlying RocksDB handle (for advanced usage) + pub(crate) fn rocks(&self) -> &RocksDB { + &self.inner + } + + /// Get a new unique collection ID for nested collections + pub fn new_collection_id(&self) -> Result { + let key = b"__global_meta:next_collection_id"; + + // Get current value + let current_bytes = self.rocks().get(key)?; + let current_id = match current_bytes { + Some(bytes) => { + if bytes.len() != 8 { + return Err(DurableError::Corruption("Invalid collection ID bytes size".into())); + } + let id_bytes: [u8; 8] = bytes[..8].try_into() + .map_err(|_| DurableError::Corruption("Invalid collection ID bytes".into()))?; + u64::from_le_bytes(id_bytes) + } + None => 0, + }; + + let next_id = current_id + 1; + + // Try to atomically update - use compare-and-swap semantics + let mut batch = WriteBatch::default(); + batch.put(key, &next_id.to_le_bytes()); + + // For now, just write it directly. In a real implementation, + // we'd want proper compare-and-swap to handle concurrent access + self.rocks().write(batch)?; + self.rocks().flush_wal(true)?; + + Ok(current_id) + } +} + +/// A write batch for atomic operations +pub struct Batch { + db: Db, + inner: WriteBatch, +} + +impl Batch { + /// Commit all operations in this batch atomically + pub fn commit(self) -> Result<()> { + self.db.rocks().write(self.inner)?; + self.db.rocks().flush_wal(true)?; + Ok(()) + } + +} diff --git a/durable/src/map.rs b/durable/src/map.rs new file mode 100644 index 0000000000000000000000000000000000000000..2629857270b1be346a20fc67dec1cfc57c829403 --- /dev/null +++ b/durable/src/map.rs @@ -0,0 +1,1073 @@ +use crate::{Db, Result, DurableError, DurableCollection}; +use rocksdb::{IteratorMode, WriteBatch, Direction}; +use serde::{Serialize, Deserialize}; +use std::marker::PhantomData; + +/// A persistent map backed by RocksDB +pub struct DurableMap { + db: Db, + prefix: Vec, + _phantom: PhantomData<(K, V)>, +} + +impl DurableMap +where + K: Serialize + for<'de> Deserialize<'de>, + V: Serialize + for<'de> Deserialize<'de>, +{ + /// Create a new DurableMap with the given name + pub fn new(db: &Db, name: &str) -> Result { + let prefix = format!("map:{}", name).into_bytes(); + + Ok(DurableMap { + db: db.clone(), + prefix, + _phantom: PhantomData, + }) + } + + /// Insert a key-value pair into the map + pub fn insert(&mut self, key: K, value: V) -> Result> { + let key_bytes = bincode::serialize(&key)?; + let value_bytes = bincode::serialize(&value)?; + + // Get the old value if it exists + let old_value = self.get(&key)?; + + let mut batch = WriteBatch::default(); + + // Write the new value + let db_key = self.entry_key(&key_bytes); + batch.put(&db_key, &value_bytes); + + // Update length if this is a new key + if old_value.is_none() { + let new_len = self.len()? + 1; + let len_key = self.meta_key("len"); + batch.put(&len_key, &(new_len as u64).to_le_bytes()); + } + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(old_value) + } + + /// Put a key-value pair into the map without returning the old value + /// + /// This is more efficient than `insert` when you don't need the old value, + /// as it only checks for key existence without deserializing the value. + pub fn put(&mut self, key: K, value: V) -> Result<()> { + let key_bytes = bincode::serialize(&key)?; + let value_bytes = bincode::serialize(&value)?; + let db_key = self.entry_key(&key_bytes); + + let mut batch = WriteBatch::default(); + + // Check if this is a new key (without deserializing the value) + let is_new = self.db.rocks().get_pinned(&db_key)?.is_none(); + + // Write the new value + batch.put(&db_key, &value_bytes); + + // Update length if this is a new key + if is_new { + let new_len = self.len()? + 1; + let len_key = self.meta_key("len"); + batch.put(&len_key, &(new_len as u64).to_le_bytes()); + } + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(()) + } + + /// Get a value by key + pub fn get(&self, key: &K) -> Result> { + let key_bytes = bincode::serialize(key)?; + let db_key = self.entry_key(&key_bytes); + + match self.db.rocks().get(&db_key)? { + Some(bytes) => { + let value = bincode::deserialize(&bytes)?; + Ok(Some(value)) + } + None => Ok(None), + } + } + + /// Check if a key exists in the map + pub fn contains_key(&self, key: &K) -> Result { + let key_bytes = bincode::serialize(key)?; + let db_key = self.entry_key(&key_bytes); + + Ok(self.db.rocks().get(&db_key)?.is_some()) + } + + /// Remove a key-value pair from the map + pub fn remove(&mut self, key: &K) -> Result> { + let key_bytes = bincode::serialize(key)?; + let db_key = self.entry_key(&key_bytes); + + // Get the old value + let old_value = match self.db.rocks().get(&db_key)? { + Some(bytes) => { + let value = bincode::deserialize(&bytes)?; + Some(value) + } + None => None, + }; + + // Delete the key if it existed and update length + if old_value.is_some() { + let mut batch = WriteBatch::default(); + + // Delete the entry + batch.delete(&db_key); + + // Update length + let new_len = self.len()? - 1; + let len_key = self.meta_key("len"); + batch.put(&len_key, &(new_len as u64).to_le_bytes()); + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + } + + Ok(old_value) + } + + + + /// Clear all entries from the map + pub fn clear(&mut self) -> Result<()> { + let prefix = self.entry_prefix(); + let mut batch = WriteBatch::default(); + + // Collect all keys to delete + let iter = self.db.rocks().iterator(IteratorMode::From(&prefix, Direction::Forward)); + for item in iter { + let (key, _) = item?; + if !key.starts_with(&prefix) { + break; + } + batch.delete(&key); + } + + // Reset length to 0 + let len_key = self.meta_key("len"); + batch.delete(&len_key); + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(()) + } + + /// Iterate over all key-value pairs using a streaming iterator + pub fn iter(&self) -> MapIterator<'_, K, V> { + let prefix = self.entry_prefix(); + let iter = self.db.rocks().iterator(IteratorMode::From(&prefix, Direction::Forward)); + + MapIterator { + inner: iter, + prefix, + _phantom: PhantomData, + } + } + + /// Load all key-value pairs into a Vec + /// + /// Note: This loads the entire collection into memory. For large collections, + /// prefer using `iter()` which streams elements. + pub fn to_vec(&self) -> Result> { + let mut result = Vec::new(); + for item in self.iter() { + result.push(item?); + } + Ok(result) + } + + /// Iterate over all keys using a streaming iterator + pub fn keys(&self) -> KeyIterator<'_, K, V> { + let prefix = self.entry_prefix(); + let iter = self.db.rocks().iterator(IteratorMode::From(&prefix, Direction::Forward)); + + KeyIterator { + inner: iter, + prefix, + _phantom: PhantomData, + } + } + + /// Load all keys into a Vec + /// + /// Note: This loads all keys into memory. For large collections, + /// prefer using `keys()` which streams elements. + pub fn keys_vec(&self) -> Result> { + let mut result = Vec::new(); + for item in self.keys() { + result.push(item?); + } + Ok(result) + } + + /// Iterate over all values using a streaming iterator + pub fn values(&self) -> ValueIterator<'_, K, V> { + let prefix = self.entry_prefix(); + let iter = self.db.rocks().iterator(IteratorMode::From(&prefix, Direction::Forward)); + + ValueIterator { + inner: iter, + prefix, + _phantom: PhantomData, + } + } + + /// Load all values into a Vec + /// + /// Note: This loads all values into memory. For large collections, + /// prefer using `values()` which streams elements. + pub fn values_vec(&self) -> Result> { + let mut result = Vec::new(); + for item in self.values() { + result.push(item?); + } + Ok(result) + } + + /// Insert multiple key-value pairs in a single batch + pub fn extend(&mut self, iter: I) -> Result<()> + where + I: IntoIterator + { + let mut batch = WriteBatch::default(); + let current_len = self.len()?; + let mut new_entries = 0; + + for (key, value) in iter { + let key_bytes = bincode::serialize(&key)?; + let value_bytes = bincode::serialize(&value)?; + let db_key = self.entry_key(&key_bytes); + + // Check if this is a new key + if !self.contains_key(&key)? { + new_entries += 1; + } + + batch.put(&db_key, &value_bytes); + } + + // Update length if we added new entries + if new_entries > 0 { + let new_len = current_len + new_entries; + let len_key = self.meta_key("len"); + batch.put(&len_key, &(new_len as u64).to_le_bytes()); + } + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(()) + } + + + + // Helper methods + + fn entry_key(&self, key_bytes: &[u8]) -> Vec { + let mut db_key = self.prefix.clone(); + db_key.extend_from_slice(b":entry:"); + db_key.extend_from_slice(key_bytes); + db_key + } + + fn entry_prefix(&self) -> Vec { + let mut prefix = self.prefix.clone(); + prefix.extend_from_slice(b":entry:"); + prefix + } +} + +// Additional implementation block for methods that don't require serialization constraints +impl DurableMap { + /// Create a new DurableMap for nested collections (no serialization constraints) + pub fn new_nested(db: &Db, name: &str) -> Self { + let prefix = format!("map:{}", name).into_bytes(); + + DurableMap { + db: db.clone(), + prefix, + _phantom: PhantomData, + } + } + + /// Create a new DurableMap from a prefix (used for nested collections) + pub fn from_prefix(db: Db, prefix: Vec) -> Self { + Self { + db, + prefix, + _phantom: PhantomData, + } + } + + /// Get the number of entries in the map (unconstrained version for nested collections) + pub fn len(&self) -> Result { + let key = self.meta_key("len"); + match self.db.rocks().get(&key)? { + Some(bytes) => { + if bytes.len() != 8 { + return Err(DurableError::Corruption("Invalid length bytes size".into())); + } + let len_bytes: [u8; 8] = bytes[..8].try_into() + .map_err(|_| DurableError::Corruption("Invalid length bytes".into()))?; + Ok(u64::from_le_bytes(len_bytes) as usize) + } + None => Ok(0), + } + } + + /// Check if the map is empty (unconstrained version for nested collections) + pub fn is_empty(&self) -> Result { + Ok(self.len()? == 0) + } + + /// Get the entry key for nested collections (needed by entry API) + pub(crate) fn make_entry_key(&self, key_bytes: &[u8]) -> Vec { + let mut db_key = self.prefix.clone(); + db_key.extend_from_slice(b":entry:"); + db_key.extend_from_slice(key_bytes); + db_key + } + + fn meta_key(&self, meta_type: &str) -> Vec { + let mut key = self.prefix.clone(); + key.extend_from_slice(b":__meta:"); + key.extend_from_slice(meta_type.as_bytes()); + key + } +} + +// Implement the DurableCollection trait for DurableMap +// This implementation is used for nested collections and doesn't require +// serialization bounds since nested maps use collection markers, not direct serialization +impl DurableCollection for DurableMap { + fn from_prefix(db: Db, prefix: Vec) -> Self { + DurableMap::from_prefix(db, prefix) + } +} + +/// Entry API for DurableMap with nested collections +pub enum DurableEntry<'a, K, V> { + Occupied(OccupiedEntry<'a, K, V>), + Vacant(VacantEntry<'a, K, V>), +} + +impl<'a, K, V> DurableEntry<'a, K, V> +where + K: Serialize, + V: DurableCollection, +{ + /// Gets the collection, creating it if it doesn't exist + pub fn or_default(self) -> Result { + match self { + DurableEntry::Occupied(entry) => entry.or_default(), + DurableEntry::Vacant(entry) => entry.or_default(), + } + } +} + +/// Represents an entry that already exists +pub struct OccupiedEntry<'a, K, V> { + map: &'a DurableMap, + key_bytes: Vec, + value_marker: Vec, // The bytes read from RocksDB, e.g., [0x02, ...] +} + +impl<'a, K, V> OccupiedEntry<'a, K, V> +where + V: DurableCollection, +{ + /// Gets a handle to the existing nested collection + pub fn get(self) -> Result { + // 1. Parse the collection_id from self.value_marker + if self.value_marker.len() != 9 || self.value_marker[0] != 0x02 { + return Err(DurableError::Corruption("Invalid collection marker".into())); + } + + let col_id_bytes: [u8; 8] = self.value_marker[1..9].try_into() + .map_err(|_| DurableError::Corruption("Invalid collection ID".into()))?; + let col_id = u64::from_le_bytes(col_id_bytes); + + // 2. Re-construct the unique prefix for the child collection + let parent_key_prefix = self.map.make_entry_key(&self.key_bytes); + let child_prefix = [parent_key_prefix.as_slice(), &[0x00], &col_id.to_le_bytes()].concat(); + + // 3. Create the collection handle using the trait method + Ok(V::from_prefix(self.map.db.clone(), child_prefix)) + } + + /// Gets a handle to the existing nested collection (same as get but consumes self) + pub fn or_default(self) -> Result { + self.get() + } +} + +/// Represents a slot that is empty +pub struct VacantEntry<'a, K, V> { + map: &'a DurableMap, + key: K, // The original key from the user +} + +impl<'a, K, V> VacantEntry<'a, K, V> +where + K: Serialize, + V: DurableCollection, +{ + /// Inserts a new default collection and returns a handle to it + pub fn or_default(self) -> Result { + // 1. Atomically get a new unique ID for the collection + let new_col_id = self.map.db.new_collection_id()?; + + // 2. Create the marker value that points to our new collection + let mut value_marker = vec![0x02_u8]; + value_marker.extend_from_slice(&new_col_id.to_le_bytes()); + + // 3. Get the key bytes and construct the full parent entry key + let key_bytes = bincode::serialize(&self.key)?; + let parent_db_key = self.map.make_entry_key(&key_bytes); + + // 4. ATOMICALLY write the marker to the parent map + self.map.db.rocks().put(&parent_db_key, &value_marker)?; + self.map.db.rocks().flush_wal(true)?; + + // 5. Construct the unique prefix for our new child collection + let child_prefix = [parent_db_key.as_slice(), &[0x00], &new_col_id.to_le_bytes()].concat(); + + // 6. Create and return the new collection handle + Ok(V::from_prefix(self.map.db.clone(), child_prefix)) + } +} + +// API for nested collections +impl DurableMap { + /// The entry point for creating or accessing a nested collection. + /// This method is only available when `V` is a `DurableCollection`. + pub fn entry(&self, key: K) -> Result> + where + V: DurableCollection, + K: Serialize, + { + let key_bytes = bincode::serialize(&key)?; + let db_key = self.make_entry_key(&key_bytes); + + match self.db.rocks().get(&db_key)? { + Some(value_marker) => { + // Key exists. The value should be a collection marker. + Ok(DurableEntry::Occupied(OccupiedEntry { + map: self, + key_bytes, + value_marker, + })) + } + None => { + // Key doesn't exist. + Ok(DurableEntry::Vacant(VacantEntry { map: self, key })) + } + } + } +} + +/// Iterator over key-value pairs in a DurableMap +pub struct MapIterator<'a, K, V> { + inner: rocksdb::DBIterator<'a>, + prefix: Vec, + _phantom: PhantomData<(K, V)>, +} + +impl<'a, K, V> Iterator for MapIterator<'a, K, V> +where + K: for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + type Item = Result<(K, V)>; + + fn next(&mut self) -> Option { + match self.inner.next() { + Some(Ok((db_key, value_bytes))) => { + // Check if we're still within our prefix + if !db_key.starts_with(&self.prefix) { + return None; + } + + // Extract the key part (skip prefix) + let key_start = self.prefix.len(); + let key_bytes = &db_key[key_start..]; + + // Deserialize key and value + match (bincode::deserialize(key_bytes), bincode::deserialize(&value_bytes)) { + (Ok(key), Ok(value)) => Some(Ok((key, value))), + (Err(e), _) | (_, Err(e)) => Some(Err(e.into())), + } + } + Some(Err(e)) => Some(Err(e.into())), + None => None, + } + } +} + +/// Iterator over keys in a DurableMap +pub struct KeyIterator<'a, K, V> { + inner: rocksdb::DBIterator<'a>, + prefix: Vec, + _phantom: PhantomData<(K, V)>, +} + +impl<'a, K, V> Iterator for KeyIterator<'a, K, V> +where + K: for<'de> Deserialize<'de>, +{ + type Item = Result; + + fn next(&mut self) -> Option { + match self.inner.next() { + Some(Ok((db_key, _))) => { + // Check if we're still within our prefix + if !db_key.starts_with(&self.prefix) { + return None; + } + + // Extract the key part (skip prefix) + let key_start = self.prefix.len(); + let key_bytes = &db_key[key_start..]; + + // Deserialize key + match bincode::deserialize(key_bytes) { + Ok(key) => Some(Ok(key)), + Err(e) => Some(Err(e.into())), + } + } + Some(Err(e)) => Some(Err(e.into())), + None => None, + } + } +} + +/// Iterator over values in a DurableMap +pub struct ValueIterator<'a, K, V> { + inner: rocksdb::DBIterator<'a>, + prefix: Vec, + _phantom: PhantomData<(K, V)>, +} + +impl<'a, K, V> Iterator for ValueIterator<'a, K, V> +where + V: for<'de> Deserialize<'de>, +{ + type Item = Result; + + fn next(&mut self) -> Option { + match self.inner.next() { + Some(Ok((db_key, value_bytes))) => { + // Check if we're still within our prefix + if !db_key.starts_with(&self.prefix) { + return None; + } + + // Deserialize value + match bincode::deserialize(&value_bytes) { + Ok(value) => Some(Ok(value)), + Err(e) => Some(Err(e.into())), + } + } + Some(Err(e)) => Some(Err(e.into())), + None => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use std::collections::HashMap; + + fn setup_test_db() -> (TempDir, Db) { + let temp_dir = TempDir::new().unwrap(); + let db = Db::open(temp_dir.path()).unwrap(); + (temp_dir, db) + } + + #[test] + fn test_insert_and_get() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "test_map").unwrap(); + + // Insert some values + assert_eq!(map.insert("one".to_string(), 1).unwrap(), None); + assert_eq!(map.insert("two".to_string(), 2).unwrap(), None); + assert_eq!(map.insert("three".to_string(), 3).unwrap(), None); + + // Get values + assert_eq!(map.get(&"one".to_string()).unwrap(), Some(1)); + assert_eq!(map.get(&"two".to_string()).unwrap(), Some(2)); + assert_eq!(map.get(&"three".to_string()).unwrap(), Some(3)); + assert_eq!(map.get(&"four".to_string()).unwrap(), None); + + // Update existing value + assert_eq!(map.insert("two".to_string(), 22).unwrap(), Some(2)); + assert_eq!(map.get(&"two".to_string()).unwrap(), Some(22)); + } + + #[test] + fn test_remove() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "remove_map").unwrap(); + + // Insert and remove + map.insert("key".to_string(), "value".to_string()).unwrap(); + assert_eq!(map.remove(&"key".to_string()).unwrap(), Some("value".to_string())); + assert_eq!(map.remove(&"key".to_string()).unwrap(), None); + assert_eq!(map.get(&"key".to_string()).unwrap(), None); + } + + #[test] + fn test_contains_key() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "contains_map").unwrap(); + + map.insert(42, "answer".to_string()).unwrap(); + + assert!(map.contains_key(&42).unwrap()); + assert!(!map.contains_key(&43).unwrap()); + } + + #[test] + fn test_len_and_clear() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "len_map").unwrap(); + + // Empty map + assert_eq!(map.len().unwrap(), 0); + assert!(map.is_empty().unwrap()); + + // Add items + for i in 0..10 { + map.insert(i, i * 2).unwrap(); + } + assert_eq!(map.len().unwrap(), 10); + assert!(!map.is_empty().unwrap()); + + // Clear + map.clear().unwrap(); + assert_eq!(map.len().unwrap(), 0); + assert!(map.is_empty().unwrap()); + } + + #[test] + fn test_persistence() { + let (temp_dir, db) = setup_test_db(); + + // Create and populate map + { + let mut map = DurableMap::>::new(&db, "persist_map").unwrap(); + map.insert("binary".to_string(), vec![1, 2, 3, 4, 5]).unwrap(); + map.insert("data".to_string(), vec![10, 20, 30]).unwrap(); + } + + // Drop the database + drop(db); + + // Reopen and verify data persists + { + let db = Db::open(temp_dir.path()).unwrap(); + let map = DurableMap::>::new(&db, "persist_map").unwrap(); + + assert_eq!(map.get(&"binary".to_string()).unwrap(), Some(vec![1, 2, 3, 4, 5])); + assert_eq!(map.get(&"data".to_string()).unwrap(), Some(vec![10, 20, 30])); + assert_eq!(map.len().unwrap(), 2); + } + } + + #[test] + fn test_iteration() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "iter_map").unwrap(); + + // Insert data + let data = vec![ + ("apple".to_string(), 1), + ("banana".to_string(), 2), + ("cherry".to_string(), 3), + ]; + + for (k, v) in &data { + map.insert(k.clone(), *v).unwrap(); + } + + // Test iter() + let mut items = map.to_vec().unwrap(); + items.sort_by_key(|(k, _)| k.clone()); + assert_eq!(items, data); + + // Test keys() + let mut keys = map.keys_vec().unwrap(); + keys.sort(); + assert_eq!(keys, vec!["apple", "banana", "cherry"]); + + // Test values() + let mut values = map.values_vec().unwrap(); + values.sort(); + assert_eq!(values, vec![1, 2, 3]); + } + + #[test] + fn test_extend() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "extend_map").unwrap(); + + // Extend from iterator + let data: HashMap = vec![ + (1, "one".to_string()), + (2, "two".to_string()), + (3, "three".to_string()), + ].into_iter().collect(); + + map.extend(data.clone()).unwrap(); + + // Verify all items were inserted + for (k, v) in data { + assert_eq!(map.get(&k).unwrap(), Some(v)); + } + assert_eq!(map.len().unwrap(), 3); + } + + #[test] + fn test_complex_keys() { + use serde::{Serialize, Deserialize}; + + #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] + struct ComplexKey { + id: u64, + name: String, + } + + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "complex_map").unwrap(); + + let key1 = ComplexKey { id: 1, name: "first".to_string() }; + let key2 = ComplexKey { id: 2, name: "second".to_string() }; + + map.insert(key1.clone(), "value1".to_string()).unwrap(); + map.insert(key2.clone(), "value2".to_string()).unwrap(); + + assert_eq!(map.get(&key1).unwrap(), Some("value1".to_string())); + assert_eq!(map.get(&key2).unwrap(), Some("value2".to_string())); + } + + #[test] + fn test_multiple_maps_same_db() { + let (_temp, db) = setup_test_db(); + + let mut map1 = DurableMap::::new(&db, "map1").unwrap(); + let mut map2 = DurableMap::::new(&db, "map2").unwrap(); + + // Insert different data + map1.insert("shared_key".to_string(), 100).unwrap(); + map2.insert("shared_key".to_string(), 200).unwrap(); + + // Verify isolation + assert_eq!(map1.get(&"shared_key".to_string()).unwrap(), Some(100)); + assert_eq!(map2.get(&"shared_key".to_string()).unwrap(), Some(200)); + } + + #[test] + fn test_streaming_iterators() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "stream_map").unwrap(); + + // Insert test data + let data = vec![ + ("alice".to_string(), 100), + ("bob".to_string(), 200), + ("charlie".to_string(), 300), + ]; + + for (k, v) in &data { + map.insert(k.clone(), *v).unwrap(); + } + + // Test streaming iteration + let mut collected = Vec::new(); + for item in map.iter() { + let (k, v) = item.unwrap(); + collected.push((k, v)); + } + collected.sort_by_key(|(k, _)| k.clone()); + assert_eq!(collected, data); + + // Test keys iterator + let mut keys = Vec::new(); + for key in map.keys() { + keys.push(key.unwrap()); + } + keys.sort(); + assert_eq!(keys, vec!["alice", "bob", "charlie"]); + + // Test values iterator + let mut values = Vec::new(); + for value in map.values() { + values.push(value.unwrap()); + } + values.sort(); + assert_eq!(values, vec![100, 200, 300]); + + // Test that iterators properly handle prefix boundaries + let mut map2 = DurableMap::::new(&db, "stream_map2").unwrap(); + map2.insert("dave".to_string(), 400).unwrap(); + + // Each iterator should only see its own data + let collected1: Vec<_> = map.iter().map(Result::unwrap).collect(); + let collected2: Vec<_> = map2.iter().map(Result::unwrap).collect(); + + assert_eq!(collected1.len(), 3); + assert_eq!(collected2.len(), 1); + assert_eq!(collected2[0], ("dave".to_string(), 400)); + } + + #[test] + fn test_metadata_length_tracking() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "length_map").unwrap(); + + // Empty map + assert_eq!(map.len().unwrap(), 0); + assert!(map.is_empty().unwrap()); + + // Insert operations should update length + map.insert("key1".to_string(), "value1".to_string()).unwrap(); + assert_eq!(map.len().unwrap(), 1); + + map.insert("key2".to_string(), "value2".to_string()).unwrap(); + assert_eq!(map.len().unwrap(), 2); + + // Updating existing key should not change length + map.insert("key1".to_string(), "new_value1".to_string()).unwrap(); + assert_eq!(map.len().unwrap(), 2); + + // Remove operations should update length + map.remove(&"key1".to_string()).unwrap(); + assert_eq!(map.len().unwrap(), 1); + + // Removing non-existent key should not change length + map.remove(&"non_existent".to_string()).unwrap(); + assert_eq!(map.len().unwrap(), 1); + + // Extend should update length correctly + let data = vec![ + ("key3".to_string(), "value3".to_string()), + ("key4".to_string(), "value4".to_string()), + ("key5".to_string(), "value5".to_string()), + ]; + map.extend(data).unwrap(); + assert_eq!(map.len().unwrap(), 4); // key2 + 3 new keys + + // Extend with existing keys should only count new ones + let mixed_data = vec![ + ("key2".to_string(), "updated_value2".to_string()), // existing + ("key6".to_string(), "value6".to_string()), // new + ]; + map.extend(mixed_data).unwrap(); + assert_eq!(map.len().unwrap(), 5); // only key6 was new + + // Clear should reset length to 0 + map.clear().unwrap(); + assert_eq!(map.len().unwrap(), 0); + assert!(map.is_empty().unwrap()); + } + + #[test] + fn test_put_method() { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "put_map").unwrap(); + + // Put new entries + map.put("a".to_string(), 1).unwrap(); + map.put("b".to_string(), 2).unwrap(); + map.put("c".to_string(), 3).unwrap(); + + // Verify entries exist and length is correct + assert_eq!(map.get(&"a".to_string()).unwrap(), Some(1)); + assert_eq!(map.get(&"b".to_string()).unwrap(), Some(2)); + assert_eq!(map.get(&"c".to_string()).unwrap(), Some(3)); + assert_eq!(map.len().unwrap(), 3); + + // Update existing entry with put + map.put("b".to_string(), 20).unwrap(); + assert_eq!(map.get(&"b".to_string()).unwrap(), Some(20)); + assert_eq!(map.len().unwrap(), 3); // Length should not change + + // Compare put vs insert performance characteristics + // put() doesn't return old value but is more efficient + map.put("d".to_string(), 4).unwrap(); + assert_eq!(map.len().unwrap(), 4); + + // insert() returns old value + let old = map.insert("d".to_string(), 40).unwrap(); + assert_eq!(old, Some(4)); + assert_eq!(map.len().unwrap(), 4); + } +} + +#[cfg(all(test, not(miri)))] +mod proptests { + use super::*; + use proptest::prelude::*; + use tempfile::TempDir; + use std::collections::HashMap; + + fn setup_test_db() -> (TempDir, Db) { + let temp_dir = TempDir::new().unwrap(); + let db = Db::open(temp_dir.path()).unwrap(); + (temp_dir, db) + } + + proptest! { + #[test] + fn prop_insert_get_consistency(data: HashMap) { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "prop_map").unwrap(); + + // Insert all pairs + for (k, v) in &data { + map.insert(k.clone(), *v).unwrap(); + } + + // Verify all can be retrieved + for (k, v) in &data { + prop_assert_eq!(map.get(k).unwrap(), Some(*v)); + } + + // Verify length + prop_assert_eq!(map.len().unwrap(), data.len()); + } + + #[test] + fn prop_remove_consistency(data: HashMap) { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "remove_map").unwrap(); + + // Insert all + map.extend(data.clone()).unwrap(); + + // Remove all and verify + for (k, v) in data { + prop_assert_eq!(map.remove(&k).unwrap(), Some(v)); + prop_assert_eq!(map.remove(&k).unwrap(), None); + prop_assert!(!map.contains_key(&k).unwrap()); + } + + prop_assert!(map.is_empty().unwrap()); + } + + #[test] + fn prop_clear_makes_empty(data: HashMap) { + let (_temp, db) = setup_test_db(); + let mut map = DurableMap::::new(&db, "clear_map").unwrap(); + + map.extend(data).unwrap(); + map.clear().unwrap(); + + prop_assert_eq!(map.len().unwrap(), 0); + prop_assert!(map.is_empty().unwrap()); + prop_assert_eq!(map.to_vec().unwrap(), vec![]); + } + } + + #[test] + fn test_nested_collections() { + use crate::DurableVec; + + let (_temp, db) = setup_test_db(); + + // Create a map where values are DurableVec + let users_posts: DurableMap> = DurableMap::new_nested(&db, "user_posts"); + + // Test creating nested collections through the entry API + let mut alice_posts = users_posts.entry("alice".to_string()).unwrap().or_default().unwrap(); + alice_posts.push(101).unwrap(); + alice_posts.push(102).unwrap(); + alice_posts.push(103).unwrap(); + + // Test accessing the same collection again + let alice_posts_again = users_posts.entry("alice".to_string()).unwrap().or_default().unwrap(); + assert_eq!(alice_posts_again.len().unwrap(), 3); + assert_eq!(alice_posts_again.get(0).unwrap(), Some(101)); + assert_eq!(alice_posts_again.get(1).unwrap(), Some(102)); + assert_eq!(alice_posts_again.get(2).unwrap(), Some(103)); + + // Test creating a different nested collection + let mut bob_posts = users_posts.entry("bob".to_string()).unwrap().or_default().unwrap(); + bob_posts.push(201).unwrap(); + bob_posts.push(202).unwrap(); + + // Verify isolation between nested collections + assert_eq!(alice_posts_again.len().unwrap(), 3); + assert_eq!(bob_posts.len().unwrap(), 2); + + // Test chained calls + users_posts.entry("charlie".to_string()).unwrap().or_default().unwrap().push(301).unwrap(); + let charlie_posts = users_posts.entry("charlie".to_string()).unwrap().or_default().unwrap(); + assert_eq!(charlie_posts.len().unwrap(), 1); + assert_eq!(charlie_posts.get(0).unwrap(), Some(301)); + } + + // Note: Nested DurableMap-in-DurableMap requires implementing Serialize/Deserialize + // for DurableMap, which is not straightforward since it contains database handles. + // For now, let's focus on the more common case of Map-to-Vec nesting. + + // Deep nesting with Map -> Map -> Vec also requires DurableMap serialization + // Let's skip this for now and focus on the fundamental Map -> Vec case + + #[test] + fn test_nested_collection_persistence() { + use crate::DurableVec; + + let (temp_dir, db) = setup_test_db(); + + // Create nested structure and populate it + { + let users_data: DurableMap> = DurableMap::new_nested(&db, "users"); + let mut user1_data = users_data.entry("user1".to_string()).unwrap().or_default().unwrap(); + user1_data.push("data1".to_string()).unwrap(); + user1_data.push("data2".to_string()).unwrap(); + + let mut user2_data = users_data.entry("user2".to_string()).unwrap().or_default().unwrap(); + user2_data.push("other_data".to_string()).unwrap(); + } + + // Drop the database + drop(db); + + // Reopen and verify persistence + { + let db = Db::open(temp_dir.path()).unwrap(); + let users_data: DurableMap> = DurableMap::new_nested(&db, "users"); + + let user1_data = users_data.entry("user1".to_string()).unwrap().or_default().unwrap(); + assert_eq!(user1_data.len().unwrap(), 2); + assert_eq!(user1_data.get(0).unwrap(), Some("data1".to_string())); + assert_eq!(user1_data.get(1).unwrap(), Some("data2".to_string())); + + let user2_data = users_data.entry("user2".to_string()).unwrap().or_default().unwrap(); + assert_eq!(user2_data.len().unwrap(), 1); + assert_eq!(user2_data.get(0).unwrap(), Some("other_data".to_string())); + } + } +} \ No newline at end of file diff --git a/durable/src/vec.rs b/durable/src/vec.rs new file mode 100644 index 0000000000000000000000000000000000000000..28d08fd0786df241aaf9c01b279708e547e4c034 --- /dev/null +++ b/durable/src/vec.rs @@ -0,0 +1,658 @@ +use crate::{Db, Result, DurableError, DurableCollection}; +use rocksdb::WriteBatch; +use serde::{Serialize, Deserialize}; +use std::marker::PhantomData; + +/// A persistent vector backed by RocksDB +pub struct DurableVec { + db: Db, + prefix: Vec, + _phantom: PhantomData, +} + +impl DurableVec +where + T: Serialize + for<'de> Deserialize<'de> +{ + /// Create a new DurableVec with the given name + pub fn new(db: &Db, name: &str) -> Result { + let prefix = format!("vec:{}", name).into_bytes(); + + Ok(DurableVec { + db: db.clone(), + prefix, + _phantom: PhantomData, + }) + } + + /// Create a new DurableVec from a prefix (used for nested collections) + pub fn from_prefix(db: Db, prefix: Vec) -> Self { + Self { + db, + prefix, + _phantom: PhantomData, + } + } + + /// Get the length of the vector + pub fn len(&self) -> Result { + let key = self.meta_key("len"); + match self.db.rocks().get(&key)? { + Some(bytes) => { + if bytes.len() != 8 { + return Err(DurableError::Corruption("Invalid length bytes size".into())); + } + let len_bytes: [u8; 8] = bytes[..8].try_into() + .map_err(|_| DurableError::Corruption("Invalid length bytes".into()))?; + Ok(u64::from_le_bytes(len_bytes) as usize) + } + None => Ok(0), + } + } + + /// Check if the vector is empty + pub fn is_empty(&self) -> Result { + Ok(self.len()? == 0) + } + + /// Push an element to the end of the vector + pub fn push(&mut self, value: T) -> Result<()> { + let len = self.len()?; + let mut batch = WriteBatch::default(); + + // Serialize the value + let value_bytes = bincode::serialize(&value)?; + + // Write the element + let elem_key = self.element_key(len); + batch.put(&elem_key, &value_bytes); + + // Update the length + let new_len = (len + 1) as u64; + let len_key = self.meta_key("len"); + batch.put(&len_key, &new_len.to_le_bytes()); + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(()) + } + + /// Get an element at the given index + pub fn get(&self, index: usize) -> Result> { + let len = self.len()?; + if index >= len { + return Ok(None); + } + + let key = self.element_key(index); + match self.db.rocks().get(&key)? { + Some(bytes) => { + let value = bincode::deserialize(&bytes)?; + Ok(Some(value)) + } + None => Err(DurableError::Corruption( + format!("Element at index {} not found but index < len", index) + )), + } + } + + /// Clear all elements from the vector + pub fn clear(&mut self) -> Result<()> { + let len = self.len()?; + let mut batch = WriteBatch::default(); + + // Delete all elements + for i in 0..len { + let key = self.element_key(i); + batch.delete(&key); + } + + // Delete the length meta key + let len_key = self.meta_key("len"); + batch.delete(&len_key); + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(()) + } + + /// Create a streaming iterator over the vector + pub fn iter(&self) -> Result> + '_> { + let prefix = self.element_prefix(); + let iter = self.db.rocks().iterator(rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward)); + + Ok(VecIterator { + inner: iter, + prefix, + _phantom: PhantomData, + }) + } + + /// Convert the entire vector to a Vec in memory + /// + /// Note: This loads the entire collection into memory. For large collections, + /// prefer using `iter()` which streams elements. + pub fn to_vec(&self) -> Result> { + let len = self.len()?; + let mut result = Vec::with_capacity(len); + + for item in self.iter()? { + result.push(item?); + } + + Ok(result) + } + + /// Push multiple elements in a single batch + pub fn extend(&mut self, iter: I) -> Result<()> + where + I: IntoIterator + { + let mut batch = WriteBatch::default(); + let mut len = self.len()?; + + for value in iter { + let value_bytes = bincode::serialize(&value)?; + let elem_key = self.element_key(len); + batch.put(&elem_key, &value_bytes); + len += 1; + } + + // Update length + let len_key = self.meta_key("len"); + batch.put(&len_key, &(len as u64).to_le_bytes()); + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(()) + } + + /// Remove and return the last element + pub fn pop(&mut self) -> Result> { + let len = self.len()?; + if len == 0 { + return Ok(None); + } + + let last_idx = len - 1; + let value = self.get(last_idx)?; + + let mut batch = WriteBatch::default(); + + // Delete the last element + let elem_key = self.element_key(last_idx); + batch.delete(&elem_key); + + // Update length + let len_key = self.meta_key("len"); + batch.put(&len_key, &(last_idx as u64).to_le_bytes()); + + // Commit atomically + self.db.rocks().write(batch)?; + self.db.rocks().flush_wal(true)?; + + Ok(value) + } + + // Helper methods + + fn element_key(&self, index: usize) -> Vec { + let mut key = self.prefix.clone(); + key.push(b':'); + key.extend_from_slice(&(index as u64).to_be_bytes()); + key + } + + fn meta_key(&self, meta_type: &str) -> Vec { + let mut key = self.prefix.clone(); + key.extend_from_slice(b":__meta:"); + key.extend_from_slice(meta_type.as_bytes()); + key + } + + fn element_prefix(&self) -> Vec { + let mut prefix = self.prefix.clone(); + prefix.push(b':'); + prefix + } +} + +// Implement the DurableCollection trait for DurableVec +impl DurableCollection for DurableVec +where + T: Serialize + for<'de> Deserialize<'de> +{ + fn from_prefix(db: Db, prefix: Vec) -> Self { + DurableVec::from_prefix(db, prefix) + } +} + +/// Iterator over a DurableVec +pub struct VecIterator<'a, T> { + inner: rocksdb::DBIterator<'a>, + prefix: Vec, + _phantom: PhantomData, +} + +impl<'a, T> Iterator for VecIterator<'a, T> +where + T: for<'de> Deserialize<'de> +{ + type Item = Result; + + fn next(&mut self) -> Option { + loop { + match self.inner.next() { + Some(Ok((key, value))) => { + // Check if we're still within our prefix + if !key.starts_with(&self.prefix) { + return None; + } + + // Check if this is a meta key (skip it) + // The key pattern is: prefix:element_index or prefix:__meta:type + // We want to skip any key that contains "__meta:" + if key.windows(7).any(|w| w == b"__meta:") { + continue; // Skip this key and try the next one + } + + // Deserialize the value + match bincode::deserialize(&value) { + Ok(item) => return Some(Ok(item)), + Err(e) => return Some(Err(e.into())), + } + } + Some(Err(e)) => return Some(Err(e.into())), + None => return None, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn setup_test_db() -> (TempDir, Db) { + let temp_dir = TempDir::new().unwrap(); + let db = Db::open(temp_dir.path()).unwrap(); + (temp_dir, db) + } + + #[test] + fn test_push_and_get() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "test_vec").unwrap(); + + // Push some values + vec.push("first".to_string()).unwrap(); + vec.push("second".to_string()).unwrap(); + vec.push("third".to_string()).unwrap(); + + // Check length + assert_eq!(vec.len().unwrap(), 3); + + // Get values + assert_eq!(vec.get(0).unwrap(), Some("first".to_string())); + assert_eq!(vec.get(1).unwrap(), Some("second".to_string())); + assert_eq!(vec.get(2).unwrap(), Some("third".to_string())); + assert_eq!(vec.get(3).unwrap(), None); + } + + #[test] + fn test_persistence() { + let (temp_dir, db) = setup_test_db(); + + // Create and populate vector + { + let mut vec = DurableVec::::new(&db, "persist_vec").unwrap(); + vec.push(42).unwrap(); + vec.push(100).unwrap(); + vec.push(-7).unwrap(); + } + + // Drop the database + drop(db); + + // Reopen and verify data persists + { + let db = Db::open(temp_dir.path()).unwrap(); + let vec = DurableVec::::new(&db, "persist_vec").unwrap(); + + assert_eq!(vec.len().unwrap(), 3); + assert_eq!(vec.get(0).unwrap(), Some(42)); + assert_eq!(vec.get(1).unwrap(), Some(100)); + assert_eq!(vec.get(2).unwrap(), Some(-7)); + } + } + + #[test] + fn test_clear() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "clear_vec").unwrap(); + + // Add some elements + vec.extend(vec![1, 2, 3, 4, 5]).unwrap(); + assert_eq!(vec.len().unwrap(), 5); + + // Clear + vec.clear().unwrap(); + assert_eq!(vec.len().unwrap(), 0); + assert!(vec.is_empty().unwrap()); + + // Should be able to push again + vec.push(42).unwrap(); + assert_eq!(vec.len().unwrap(), 1); + assert_eq!(vec.get(0).unwrap(), Some(42)); + } + + #[test] + fn test_pop() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "pop_vec").unwrap(); + + // Empty pop + assert_eq!(vec.pop().unwrap(), None); + + // Push and pop + vec.push("a".to_string()).unwrap(); + vec.push("b".to_string()).unwrap(); + vec.push("c".to_string()).unwrap(); + + assert_eq!(vec.pop().unwrap(), Some("c".to_string())); + assert_eq!(vec.len().unwrap(), 2); + assert_eq!(vec.pop().unwrap(), Some("b".to_string())); + assert_eq!(vec.len().unwrap(), 1); + assert_eq!(vec.pop().unwrap(), Some("a".to_string())); + assert_eq!(vec.len().unwrap(), 0); + assert_eq!(vec.pop().unwrap(), None); + } + + #[test] + fn test_iteration() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "iter_vec").unwrap(); + + // Add elements + let values = vec![10, 20, 30, 40, 50]; + vec.extend(values.clone()).unwrap(); + + // Iterate and collect + let collected = vec.to_vec().unwrap(); + + assert_eq!(collected, values); + } + + #[test] + fn test_extend() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "extend_vec").unwrap(); + + // Extend with iterator + vec.extend(vec!["a", "b", "c"].into_iter().map(String::from)).unwrap(); + assert_eq!(vec.len().unwrap(), 3); + + // Extend again + vec.extend(vec!["d", "e"].into_iter().map(String::from)).unwrap(); + assert_eq!(vec.len().unwrap(), 5); + + // Verify all elements + let all = vec.to_vec().unwrap(); + assert_eq!(all, vec!["a", "b", "c", "d", "e"]); + } + + #[test] + fn test_large_dataset() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "large_vec").unwrap(); + + // Push many elements + let count = 1000; + for i in 0..count { + vec.push(i).unwrap(); + } + + assert_eq!(vec.len().unwrap(), count as usize); + + // Verify some random accesses + assert_eq!(vec.get(0).unwrap(), Some(0)); + assert_eq!(vec.get(500).unwrap(), Some(500)); + assert_eq!(vec.get(999).unwrap(), Some(999)); + assert_eq!(vec.get(1000).unwrap(), None); + + // Verify iteration count + let all_values = vec.to_vec().unwrap(); + assert_eq!(all_values.len(), count as usize); + } + + #[test] + fn test_complex_types() { + use serde::{Serialize, Deserialize}; + + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] + struct User { + id: u64, + name: String, + email: String, + active: bool, + } + + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "users").unwrap(); + + let user1 = User { + id: 1, + name: "Alice".to_string(), + email: "alice@example.com".to_string(), + active: true, + }; + + let user2 = User { + id: 2, + name: "Bob".to_string(), + email: "bob@example.com".to_string(), + active: false, + }; + + vec.push(user1.clone()).unwrap(); + vec.push(user2.clone()).unwrap(); + + assert_eq!(vec.get(0).unwrap(), Some(user1)); + assert_eq!(vec.get(1).unwrap(), Some(user2)); + } + + #[test] + fn test_empty_vec_operations() { + let (_temp, db) = setup_test_db(); + let vec = DurableVec::::new(&db, "empty_vec").unwrap(); + + // Test operations on empty vec + assert_eq!(vec.len().unwrap(), 0); + assert!(vec.is_empty().unwrap()); + assert_eq!(vec.get(0).unwrap(), None); + assert_eq!(vec.get(100).unwrap(), None); + assert_eq!(vec.to_vec().unwrap(), Vec::::new()); + } + + #[test] + fn test_multiple_vecs_same_db() { + let (_temp, db) = setup_test_db(); + + // Create multiple vectors with different names + let mut vec1 = DurableVec::::new(&db, "vec1").unwrap(); + let mut vec2 = DurableVec::::new(&db, "vec2").unwrap(); + + // Push different data to each + vec1.push("vec1_data".to_string()).unwrap(); + vec2.push("vec2_data".to_string()).unwrap(); + + // Verify they don't interfere + assert_eq!(vec1.get(0).unwrap(), Some("vec1_data".to_string())); + assert_eq!(vec2.get(0).unwrap(), Some("vec2_data".to_string())); + assert_eq!(vec1.len().unwrap(), 1); + assert_eq!(vec2.len().unwrap(), 1); + } + + #[test] + fn test_batch_atomicity() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "batch_vec").unwrap(); + + // Add initial data + vec.push(1).unwrap(); + vec.push(2).unwrap(); + vec.push(3).unwrap(); + + // Verify initial state + assert_eq!(vec.len().unwrap(), 3); + + // Clear should be atomic - either all elements deleted or none + vec.clear().unwrap(); + assert_eq!(vec.len().unwrap(), 0); + + // Extend should be atomic - either all elements added or none + vec.extend(vec![10, 20, 30, 40, 50]).unwrap(); + assert_eq!(vec.len().unwrap(), 5); + let all = vec.to_vec().unwrap(); + assert_eq!(all, vec![10, 20, 30, 40, 50]); + } + + #[test] + fn test_unicode_strings() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "unicode_vec").unwrap(); + + let test_strings = vec![ + "Hello, 世界!".to_string(), + "🦀 Rust 🚀".to_string(), + "Ñoño".to_string(), + "🏴‍☠️ Pirates".to_string(), + ]; + + vec.extend(test_strings.clone()).unwrap(); + + let retrieved = vec.to_vec().unwrap(); + assert_eq!(retrieved, test_strings); + } + + #[test] + fn test_streaming_iterator() { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "stream_vec").unwrap(); + + // Add test data + let values = vec![1, 2, 3, 4, 5]; + vec.extend(values.clone()).unwrap(); + + // Test streaming iteration + let mut collected = Vec::new(); + for item in vec.iter().unwrap() { + collected.push(item.unwrap()); + } + + assert_eq!(collected, values); + + // Test that iterator properly handles prefix boundaries + let mut vec2 = DurableVec::::new(&db, "stream_vec2").unwrap(); + vec2.extend(vec![10, 20, 30]).unwrap(); + + // Each iterator should only see its own data + let collected1: Vec<_> = vec.iter().unwrap().collect::>>().unwrap(); + let collected2: Vec<_> = vec2.iter().unwrap().collect::>>().unwrap(); + + assert_eq!(collected1, values); + assert_eq!(collected2, vec![10, 20, 30]); + } +} + +#[cfg(all(test, not(miri)))] // Skip proptest under miri +mod proptests { + use super::*; + use proptest::prelude::*; + use tempfile::TempDir; + + fn setup_test_db() -> (TempDir, Db) { + let temp_dir = TempDir::new().unwrap(); + let db = Db::open(temp_dir.path()).unwrap(); + (temp_dir, db) + } + + proptest! { + #[test] + fn prop_push_get_consistency(values: Vec) { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "prop_vec").unwrap(); + + // Push all values + for value in &values { + vec.push(*value).unwrap(); + } + + // Verify length + prop_assert_eq!(vec.len().unwrap(), values.len()); + + // Verify all values can be retrieved correctly + for (i, expected) in values.iter().enumerate() { + prop_assert_eq!(vec.get(i).unwrap(), Some(*expected)); + } + } + + #[test] + fn prop_extend_iter_roundtrip(values: Vec) { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "extend_vec").unwrap(); + + // Extend with all values + vec.extend(values.clone()).unwrap(); + + // Get back via iteration + let retrieved = vec.to_vec().unwrap(); + + prop_assert_eq!(retrieved, values); + } + + #[test] + fn prop_pop_removes_last(mut values: Vec) { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "pop_vec").unwrap(); + + // Add all values + vec.extend(values.clone()).unwrap(); + + // Pop values and verify + while let Some(expected) = values.pop() { + let popped = vec.pop().unwrap(); + prop_assert_eq!(popped, Some(expected)); + prop_assert_eq!(vec.len().unwrap(), values.len()); + } + + // Vector should be empty + prop_assert!(vec.is_empty().unwrap()); + prop_assert_eq!(vec.pop().unwrap(), None); + } + + #[test] + fn prop_clear_makes_empty(values: Vec) { + let (_temp, db) = setup_test_db(); + let mut vec = DurableVec::::new(&db, "clear_vec").unwrap(); + + // Add values + vec.extend(values).unwrap(); + + // Clear + vec.clear().unwrap(); + + // Should be empty + prop_assert_eq!(vec.len().unwrap(), 0); + prop_assert!(vec.is_empty().unwrap()); + prop_assert_eq!(vec.get(0).unwrap(), None); + } + } +} \ No newline at end of file diff --git a/durable/todo.tdsl b/durable/todo.tdsl new file mode 100644 index 0000000000000000000000000000000000000000..6c9df9cbb25bb72d17aac07296a0c935ccbe89e7 --- /dev/null +++ b/durable/todo.tdsl @@ -0,0 +1,4 @@ +Add streaming iterators to avoid loading entire collections +Implement collection nesting (e.g., DurableMap>) +Add benchmarks to measure performance +Implement schema versioning and migrations diff --git a/server/Cargo.toml b/server/Cargo.toml index 6fb7bf52fa58f46f5e8fb0f7fd395247b57506d7..34672bd7acb9d165eea290e779c548fa4d2b8e3d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -22,6 +22,7 @@ async-stream = "0.3" futures-util = { version = "0.3", default-features = false, features = ["std"] } rand = "0.8" urlencoding = "2" +durable = { path = "../durable" } [dev-dependencies] reqwest = { version = "0.12", features = ["json"] } diff --git a/server/src/entity_store.rs b/server/src/entity_store.rs new file mode 100644 index 0000000000000000000000000000000000000000..be3300529e97801a4d42f06f3f669b78ddd6856d --- /dev/null +++ b/server/src/entity_store.rs @@ -0,0 +1,90 @@ +//! Off-heap storage for full entity payloads (Reddit API JSON). +//! +//! Derived [`crate::reducer::EntityData`] stays in the in-memory tree; raw JSON +//! lives in RocksDB via the workspace `durable` crate. + +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use durable::{Db, DurableMap}; +use serde_json::Value; + +use crate::path_types::ItemId; + +#[derive(Debug, thiserror::Error)] +pub enum EntityStoreError { + #[error("durable error: {0}")] + Durable(#[from] durable::DurableError), + #[error("json error: {0}")] + Json(#[from] serde_json::Error), + #[error("io error: {0}")] + Io(#[from] std::io::Error), + #[error("store lock poisoned")] + Poisoned, +} + +struct EntityStoreInner { + _db: Db, + payloads: DurableMap, +} + +/// Disk-backed map of entity id → raw JSON payload. +#[derive(Clone)] +pub struct EntityStore { + inner: Arc>, +} + +impl EntityStore { + /// Open (or create) the entity database under `dir`. + pub fn open(dir: &Path) -> Result { + std::fs::create_dir_all(dir)?; + let db = Db::open(dir)?; + let payloads = DurableMap::new(&db, "entity_payloads")?; + Ok(Self { + inner: Arc::new(Mutex::new(EntityStoreInner { _db: db, payloads })), + }) + } + + /// Persist a payload for `id` (overwrites any existing entry). + pub fn put(&self, id: &ItemId, payload: &Value) -> Result<(), EntityStoreError> { + let json = serde_json::to_string(payload)?; + let mut inner = self + .inner + .lock() + .map_err(|_| EntityStoreError::Poisoned)?; + inner + .payloads + .put(id.as_str().to_string(), json) + .map_err(EntityStoreError::from) + } + + /// Load a stored payload, if present. + pub fn get(&self, id: &ItemId) -> Result, EntityStoreError> { + let inner = self + .inner + .lock() + .map_err(|_| EntityStoreError::Poisoned)?; + match inner.payloads.get(&id.as_str().to_string())? { + Some(json) => Ok(Some(serde_json::from_str(&json)?)), + None => Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn round_trip_payload() { + let tmp = tempfile::tempdir().unwrap(); + let store = EntityStore::open(tmp.path()).unwrap(); + let id = ItemId::parse("reddit.com/r/rust").unwrap(); + let payload = json!({"kind": "t5", "data": {"display_name": "rust"}}); + + store.put(&id, &payload).unwrap(); + let loaded = store.get(&id).unwrap().unwrap(); + assert_eq!(loaded, payload); + } +} diff --git a/server/src/event_log.rs b/server/src/event_log.rs index 03c86f7f028cdf167b4943951d724da486b1b82c..838c8830b265f28a03442681575271eef539508c 100644 --- a/server/src/event_log.rs +++ b/server/src/event_log.rs @@ -7,6 +7,12 @@ use tokio::{ use crate::events::Event; +#[derive(Debug, Default)] +pub struct ReplayStats { + pub applied: usize, + pub bad_lines: usize, +} + #[derive(Debug, thiserror::Error)] pub enum EventLogError { #[error("io error: {0}")] @@ -51,30 +57,87 @@ impl EventLog { Ok(()) } - pub async fn load_all(&self) -> Result<(Vec, Vec<(usize, String)>), EventLogError> { + /// Stream the log one line at a time — parse each [`Event`], apply, drop before the next line. + pub async fn replay(&self, mut apply: F) -> Result + where + F: FnMut(Event) -> Result<(), EventLogError>, + { + let mut stats = ReplayStats::default(); if !fs::try_exists(&self.path).await? { - return Ok((vec![], vec![])); + return Ok(stats); } let f = fs::File::open(&self.path).await?; let mut reader = BufReader::new(f).lines(); - let mut events = Vec::new(); - let mut bad_lines = Vec::new(); - - let mut line_no: usize = 0; while let Some(line) = reader.next_line().await? { - line_no += 1; let trimmed = line.trim(); if trimmed.is_empty() { continue; } match serde_json::from_str::(trimmed) { - Ok(ev) => events.push(ev), - Err(_) => bad_lines.push((line_no, line)), + Ok(ev) => match apply(ev) { + Ok(()) => stats.applied += 1, + Err(e) => return Err(e), + }, + Err(_) => stats.bad_lines += 1, } } - Ok((events, bad_lines)) + Ok(stats) + } + + /// Load every event into memory. Prefer [`Self::replay`] for startup. + pub async fn load_all(&self) -> Result<(Vec, Vec<(usize, String)>), EventLogError> { + let mut events = Vec::new(); + let stats = self + .replay(|ev| { + events.push(ev); + Ok(()) + }) + .await?; + let _ = stats; + Ok((events, vec![])) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::Event; + + #[tokio::test] + async fn replay_applies_one_line_at_a_time() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("events.jsonl"); + let log = EventLog::new(&path); + log.append(&Event::NodeEnsured { + id: "reddit.com/r/rust".into(), + }) + .await + .unwrap(); + log.append(&Event::VoteRecorded { + ts: 1, + a: "a".into(), + b: "b".into(), + ratio_left: 2, + ratio_right: 1, + scope: String::new(), + }) + .await + .unwrap(); + + let mut seen = Vec::new(); + let stats = log + .replay(|ev| { + seen.push(ev); + Ok(()) + }) + .await + .unwrap(); + + assert_eq!(stats.applied, 2); + assert_eq!(stats.bad_lines, 0); + assert_eq!(seen.len(), 2); } } diff --git a/server/src/lib.rs b/server/src/lib.rs index 7f7e28c8ac3758de87f1f8e073b24be4132d38da..c9440a1019cb05c410eeb07b0b3ba09e4c6a6c6a 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,4 +1,5 @@ pub mod api; +pub mod entity_store; pub mod event_log; pub mod events; pub mod fetch; diff --git a/server/src/reddit.rs b/server/src/reddit.rs index 626454e5a2f638734193b5190a2beea286af85b6..28fcb83a2dfef01c76b802cec850624197b4b686 100644 --- a/server/src/reddit.rs +++ b/server/src/reddit.rs @@ -10,6 +10,7 @@ use serde_json::Value; use tokio::sync::{mpsc, oneshot, RwLock}; use crate::{ + entity_store::EntityStore, event_log::EventLog, events::Event, fetch::now_ms, @@ -80,6 +81,7 @@ impl RedditBroker { pub fn spawn( tree: Arc>, event_log: Arc, + entity_store: EntityStore, config: RedditApiConfig, ) -> Self { let (tx, rx) = mpsc::channel(100); @@ -104,7 +106,7 @@ impl RedditBroker { "reddit worker started" ); - tokio::spawn(reddit_worker(rx, tree, event_log, client, config)); + tokio::spawn(reddit_worker(rx, tree, event_log, entity_store, client, config)); Self { tx } } @@ -193,9 +195,16 @@ pub fn entity_view_from_payload(id: &ItemId, payload: &Value) -> Option Result<(), String> { let view = entity_view_from_payload(id, &payload); - tree.apply_entity_raw(id, payload, view); + store.put(id, &payload).map_err(|e| e.to_string())?; + tree.apply_entity(id, view); + Ok(()) } fn notify(done: Option>, result: FetchJobResult) { @@ -208,6 +217,7 @@ async fn reddit_worker( mut rx: mpsc::Receiver, tree: Arc>, event_log: Arc, + entity_store: EntityStore, client: Client, config: RedditApiConfig, ) { @@ -302,14 +312,16 @@ async fn reddit_worker( let mut tree = tree.write().await; if kind == FetchKind::Children { let view = entity_view_from_payload(&child_id, &child_payload); - tree.apply_entity_under_parent( - &fetch_id, - &child_id, - child_payload, - view, - ); - } else { - apply_entity_import(&mut tree, &child_id, child_payload); + if let Err(e) = entity_store.put(&child_id, &child_payload) { + write_err = Some(e.to_string()); + break; + } + tree.apply_entity_under_parent(&fetch_id, &child_id, view); + } else if let Err(e) = + apply_entity_import(&mut tree, &entity_store, &child_id, child_payload) + { + write_err = Some(e); + break; } } written += 1; diff --git a/server/src/reducer.rs b/server/src/reducer.rs index fc23f41137d7df33997afe19c533505c250cc305..cf00f235d02e487439f8767d532929fe329a0084 100644 --- a/server/src/reducer.rs +++ b/server/src/reducer.rs @@ -1,7 +1,6 @@ use std::collections::{HashMap, HashSet, VecDeque}; use serde::{Deserialize, Serialize}; -use serde_json::Value; use crate::path_types::ItemId; @@ -134,9 +133,8 @@ pub struct EntityData { #[derive(Debug, Clone, Default)] pub struct NodeState { pub id: ItemId, - /// Full imported API JSON (persisted in the event log). - pub entity_raw: Option, - /// Domain-specific view derived from `entity_raw` (e.g. Reddit title/author). + /// Domain-specific view derived from imported payload (e.g. Reddit title/author). + /// Raw JSON lives in [`crate::entity_store::EntityStore`]. pub data: Option, pub children: HashSet, pub local_ranking: GroupState, @@ -206,28 +204,25 @@ impl GlobalTree { } } - pub fn apply_entity_raw(&mut self, id: &ItemId, payload: Value, view: Option) { + pub fn apply_entity(&mut self, id: &ItemId, view: Option) { self.ensure_path(id); if let Some(node) = self.nodes.get_mut(id) { - node.entity_raw = Some(payload); node.data = view; } } - /// Import entity data for `id` and attach it as a direct child of `parent` + /// Import entity view for `id` and attach it as a direct child of `parent` /// without running [`Self::ensure_path`] on `id` (avoids Reddit `/comments/` /// parent rules pulling intermediate path segments into the subreddit). pub fn apply_entity_under_parent( &mut self, parent: &ItemId, id: &ItemId, - payload: Value, view: Option, ) { self.ensure_path(parent); self.ensure_node(id); if let Some(node) = self.nodes.get_mut(id) { - node.entity_raw = Some(payload); node.data = view; } if let Some(p) = self.nodes.get_mut(parent) { diff --git a/server/src/state.rs b/server/src/state.rs index 4c3008e73cc74d8483a2064a0a280e73d82a01ec..dce04022de7ad9b0ffe0bac05ea79182b616945a 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use crate::{ + entity_store::EntityStore, event_log::EventLog, events::Event, journal::JournalClient, @@ -43,6 +44,42 @@ fn parent_from_event_scope(scope: &str) -> ItemId { } } +fn apply_event( + ev: Event, + tree: &mut GlobalTree, + entity_store: &EntityStore, +) -> Result<(), crate::event_log::EventLogError> { + match ev { + Event::VoteRecorded { + ts, + a, + b, + ratio_left, + ratio_right, + scope, + } => { + if let Some(vote) = VoteData::from_recorded(ts, &a, &b, ratio_left, ratio_right) { + let parent = parent_from_event_scope(&scope); + tree.apply_vote(&parent, vote); + } + } + Event::ViewRecorded { .. } => {} + Event::NodeEnsured { id } => { + if let Some(parsed) = ItemId::parse(&id).or_else(|| ItemId::from_url(&id)) { + tree.ensure_path(&parsed); + } + } + Event::EntityImported { id, payload, .. } => { + if let Some(parsed) = ItemId::parse(&id).or_else(|| ItemId::from_url(&id)) { + if let Err(e) = apply_entity_import(tree, entity_store, &parsed, payload) { + tracing::warn!(item = %id, err = %e, "entity replay failed"); + } + } + } + } + Ok(()) +} + #[derive(Clone)] pub struct AppConfig { pub data_dir: String, @@ -71,6 +108,7 @@ impl AppConfig { pub struct AppState { pub cfg: Arc, pub event_log: Arc, + pub entity_store: EntityStore, pub views: ViewStore, pub tree: Arc>, journal: JournalClient, @@ -82,48 +120,31 @@ impl AppState { let event_log = Arc::new(EventLog::new(cfg.event_log_path.clone())); let views_path = format!("{}/views.json", cfg.data_dir); let views = ViewStore::new(&views_path); + let entity_db_path = format!("{}/entity_db", cfg.data_dir); + let entity_store = + EntityStore::open(std::path::Path::new(&entity_db_path)).expect("entity store"); let mut tree = GlobalTree::new(); - if let Ok((events, _)) = event_log.load_all().await { - for ev in events { - match ev { - Event::VoteRecorded { - ts, - a, - b, - ratio_left, - ratio_right, - scope, - } => { - if let Some(vote) = - VoteData::from_recorded(ts, &a, &b, ratio_left, ratio_right) - { - let parent = parent_from_event_scope(&scope); - tree.apply_vote(&parent, vote); - } - } - Event::ViewRecorded { .. } => {} - Event::NodeEnsured { id } => { - if let Some(parsed) = ItemId::parse(&id).or_else(|| ItemId::from_url(&id)) { - tree.ensure_path(&parsed); - } - } - Event::EntityImported { id, payload, .. } => { - if let Some(parsed) = ItemId::parse(&id).or_else(|| ItemId::from_url(&id)) { - apply_entity_import(&mut tree, &parsed, payload); - } - } - } - } + if let Err(e) = event_log + .replay(|ev| apply_event(ev, &mut tree, &entity_store)) + .await + { + tracing::warn!(err = %e, "event log replay failed"); } let tree = Arc::new(RwLock::new(tree)); let journal = JournalClient::spawn(tree.clone(), event_log.clone()); - let reddit = RedditBroker::spawn(tree.clone(), event_log.clone(), RedditApiConfig::from_env()); + let reddit = RedditBroker::spawn( + tree.clone(), + event_log.clone(), + entity_store.clone(), + RedditApiConfig::from_env(), + ); Self { cfg: Arc::new(cfg), event_log, + entity_store, views, tree, journal, @@ -184,10 +205,10 @@ impl AppState { mod tests { use super::{normalize_scope, parse_item_param}; use crate::{ + entity_store::EntityStore, event_log::EventLog, events::Event, path_types::ItemId, - reddit::apply_entity_import, reducer::GlobalTree, }; use serde_json::json; @@ -197,6 +218,7 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let log_path = tmp.path().join("events.jsonl"); let log = EventLog::new(log_path.to_string_lossy().into_owned()); + let entity_store = EntityStore::open(&tmp.path().join("entity_db")).unwrap(); let payload = json!({"kind":"t5","data":{"title":"Rust","display_name":"rust"}}); log.append(&Event::EntityImported { id: "reddit.com/r/rust".into(), @@ -207,19 +229,16 @@ mod tests { .unwrap(); let mut tree = GlobalTree::new(); - let (events, _) = log.load_all().await.unwrap(); - for ev in events { - if let Event::EntityImported { id, payload, .. } = ev { - let parsed = ItemId::parse(&id).unwrap(); - apply_entity_import(&mut tree, &parsed, payload); - } - } + log.replay(|ev| super::apply_event(ev, &mut tree, &entity_store)) + .await + .unwrap(); let node = tree.get(&ItemId::parse("reddit.com/r/rust").unwrap()).unwrap(); assert_eq!(node.data.as_ref().unwrap().title, "Rust"); - assert_eq!( - node.entity_raw.as_ref().unwrap()["data"]["display_name"], - "rust" - ); + let stored = entity_store + .get(&ItemId::parse("reddit.com/r/rust").unwrap()) + .unwrap() + .unwrap(); + assert_eq!(stored["data"]["display_name"], "rust"); } #[test] diff --git a/server/tests/integration_ui.rs b/server/tests/integration_ui.rs index 56d2db313b313ffce07eff38a8ccdaa2fbb9498f..32da054b58b97a4751a038c677d213d6c3fa7cdc 100644 --- a/server/tests/integration_ui.rs +++ b/server/tests/integration_ui.rs @@ -104,9 +104,17 @@ async fn post_ui_record_vote_morphs_ranking_and_persists() { let log = std::fs::read_to_string(tmp.path().join("events.jsonl")).unwrap(); assert!(log.contains("vote_recorded")); + // Replay in a fresh data dir (RocksDB locks entity_db while the server runs). + let replay_tmp = TempDir::new().unwrap(); + std::fs::copy( + tmp.path().join("events.jsonl"), + replay_tmp.path().join("events.jsonl"), + ) + .unwrap(); + let replay_data = replay_tmp.path().to_string_lossy().into_owned(); let cfg = AppConfig { - data_dir: tmp.path().to_string_lossy().into_owned(), - event_log_path: tmp.path().join("events.jsonl").to_string_lossy().into_owned(), + data_dir: replay_data.clone(), + event_log_path: format!("{replay_data}/events.jsonl"), port: 0, }; let state = create_app_state(cfg).await;