You are a constitutional council ranking individual git commits for ownership allocation. Compare these two commits. Decide which contributed more lasting value to the project. Judge substance, not spectacle: - Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise. - Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one. - Do not favor a side merely because its patch is longer or noisier. - Weight what the change does for the project, not the contributor's name. Return ONLY a JSON object: {"winner": "A" or "B", "ratio": "N:M", "explanation": "..."} The explanation must cite concrete differences in the patches (1-3 sentences). Side A — contributor: tommy-mor Side A — commit message: [432e1450] nice Side A — unified diff (full patch): diff --git a/reddit.rs b/reddit.rs new file mode 100644 index 0000000000000000000000000000000000000000..da27f6f76482dd27d44cad9d7b844972bf042e64 --- /dev/null +++ b/reddit.rs @@ -0,0 +1,420 @@ +use governor::{Quota, RateLimiter, Jitter}; +use nonzero_ext::nonzero; +use reqwest::{Client, StatusCode}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::Duration; +use anyhow::{Result, Context, anyhow}; + +/// Reddit API client with built-in rate limiting +pub struct RedditClient { + client: Client, + limiter: Arc, + user_agent: String, +} + +impl RedditClient { + /// Create a new Reddit client with rate limiting + /// Reddit API allows 60 requests per minute for OAuth2 authenticated apps + /// We'll be conservative and use 50 requests per minute + pub fn new() -> Self { + // Create rate limiter: 50 requests per minute + let quota = Quota::per_minute(nonzero!(50u32)); + let limiter = Arc::new(RateLimiter::direct(quota)); + + // Create HTTP client with timeout + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("Failed to create HTTP client"); + + // Reddit requires a unique user agent + let user_agent = format!( + "rust:sorter:v{} (by /u/hourLong_arnould)", + env!("CARGO_PKG_VERSION") + ); + + Self { + client, + limiter, + user_agent, + } + } + + /// Fetch hot posts from a subreddit + pub async fn get_subreddit_hot(&self, subreddit: &str, limit: usize) -> Result { + // Wait for rate limiter + self.limiter.until_ready_with_jitter(Jitter::up_to(Duration::from_millis(100))).await; + + let url = format!("https://www.reddit.com/r/{}/hot.json?limit={}", subreddit, limit); + + let response = self.client + .get(&url) + .header("User-Agent", &self.user_agent) + .send() + .await + .context("Failed to send request to Reddit")?; + + self.handle_response(response).await + } + + /// Fetch top posts from a subreddit + pub async fn get_subreddit_top(&self, subreddit: &str, limit: usize, time_period: &str) -> Result { + self.limiter.until_ready_with_jitter(Jitter::up_to(Duration::from_millis(100))).await; + + let url = format!( + "https://www.reddit.com/r/{}/top.json?limit={}&t={}", + subreddit, limit, time_period + ); + + let response = self.client + .get(&url) + .header("User-Agent", &self.user_agent) + .send() + .await + .context("Failed to send request to Reddit")?; + + self.handle_response(response).await + } + + /// Fetch new posts from a subreddit + pub async fn get_subreddit_new(&self, subreddit: &str, limit: usize) -> Result { + self.limiter.until_ready_with_jitter(Jitter::up_to(Duration::from_millis(100))).await; + + let url = format!("https://www.reddit.com/r/{}/new.json?limit={}", subreddit, limit); + + let response = self.client + .get(&url) + .header("User-Agent", &self.user_agent) + .send() + .await + .context("Failed to send request to Reddit")?; + + self.handle_response(response).await + } + + /// Fetch a specific post and its comments + pub async fn get_post(&self, subreddit: &str, post_id: &str) -> Result> { + self.limiter.until_ready_with_jitter(Jitter::up_to(Duration::from_millis(100))).await; + + let url = format!( + "https://www.reddit.com/r/{}/comments/{}.json", + subreddit, post_id + ); + + let response = self.client + .get(&url) + .header("User-Agent", &self.user_agent) + .send() + .await + .context("Failed to send request to Reddit")?; + + match response.status() { + StatusCode::OK => { + let listings: Vec = response + .json() + .await + .context("Failed to parse Reddit response")?; + Ok(listings) + } + StatusCode::TOO_MANY_REQUESTS => { + Err(anyhow!("Reddit rate limit exceeded. Please try again later.")) + } + StatusCode::NOT_FOUND => { + Err(anyhow!("Post not found: r/{}/comments/{}", subreddit, post_id)) + } + status => { + Err(anyhow!("Reddit API error: {}", status)) + } + } + } + + /// Fetch user profile (posts and comments) + pub async fn get_user_profile(&self, username: &str, content_type: &str, limit: usize) -> Result { + self.limiter.until_ready_with_jitter(Jitter::up_to(Duration::from_millis(100))).await; + + let url = format!( + "https://www.reddit.com/user/{}/{}.json?limit={}", + username, content_type, limit + ); + + let response = self.client + .get(&url) + .header("User-Agent", &self.user_agent) + .send() + .await + .context("Failed to send request to Reddit")?; + + self.handle_response(response).await + } + + /// Handle Reddit API response with proper error checking + async fn handle_response(&self, response: reqwest::Response) -> Result { + match response.status() { + StatusCode::OK => { + let listing: RedditListingResponse = response + .json() + .await + .context("Failed to parse Reddit response")?; + Ok(listing) + } + StatusCode::TOO_MANY_REQUESTS => { + Err(anyhow!("Reddit rate limit exceeded. Please try again later.")) + } + StatusCode::NOT_FOUND => { + Err(anyhow!("Reddit resource not found")) + } + StatusCode::FORBIDDEN => { + Err(anyhow!("Access forbidden. The subreddit may be private.")) + } + status => { + Err(anyhow!("Reddit API error: {}", status)) + } + } + } +} + +// Reddit API response types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedditListingResponse { + pub kind: String, + pub data: RedditListingData, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedditListingData { + #[serde(skip_serializing_if = "Option::is_none")] + pub after: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub before: Option, + pub children: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub modhash: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedditThing { + pub kind: String, + pub data: RedditThingData, +} + +/// Reddit's "Thing" data can be either a post, comment, or other types +/// We use untagged enum to handle different data structures +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum RedditThingData { + Post(RedditPost), + Comment(RedditComment), + // For things we don't care about yet (like "more" comments) + Other(serde_json::Value), +} + +/// Reddit post data with serde handling all the parsing +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedditPost { + pub id: String, + pub name: String, // Full name (t3_xxx) + #[serde(default = "default_untitled")] + pub title: String, + #[serde(default = "default_deleted_user")] + pub author: String, + pub subreddit: String, + #[serde(default)] + pub url: String, + #[serde(default)] + pub permalink: String, + #[serde(default, deserialize_with = "deserialize_selftext")] + pub selftext: Option, + #[serde(default)] + pub score: i64, + #[serde(default)] + pub num_comments: i64, + #[serde(default)] + pub created_utc: f64, + #[serde(default, deserialize_with = "deserialize_thumbnail")] + pub thumbnail: Option, + #[serde(default)] + pub is_video: bool, + #[serde(default)] + pub is_self: bool, + // Additional useful fields + #[serde(default)] + pub ups: i64, + #[serde(default)] + pub downs: i64, + #[serde(default)] + pub upvote_ratio: f64, + #[serde(default)] + pub over_18: bool, + #[serde(default)] + pub spoiler: bool, + #[serde(default)] + pub stickied: bool, + #[serde(default)] + pub locked: bool, + #[serde(default)] + pub distinguished: Option, +} + +/// Reddit comment data +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedditComment { + pub id: String, + pub name: String, // Full name (t1_xxx) + #[serde(default = "default_deleted_user")] + pub author: String, + #[serde(default)] + pub body: String, + #[serde(default)] + pub score: i64, + #[serde(default)] + pub created_utc: f64, + #[serde(default)] + pub parent_id: String, + #[serde(default)] + pub permalink: String, + #[serde(default)] + pub depth: i32, + // Additional useful fields + #[serde(default)] + pub ups: i64, + #[serde(default)] + pub downs: i64, + #[serde(default)] + pub edited: RedditEditStatus, + #[serde(default)] + pub stickied: bool, + #[serde(default)] + pub distinguished: Option, + #[serde(default)] + pub is_submitter: bool, + #[serde(default)] + pub collapsed: bool, + #[serde(default)] + pub controversiality: i32, +} + +/// Reddit's edit status - can be false or a timestamp +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum RedditEditStatus { + NotEdited(bool), + EditedAt(f64), +} + +impl Default for RedditEditStatus { + fn default() -> Self { + RedditEditStatus::NotEdited(false) + } +} + +// Helper functions for serde defaults +fn default_untitled() -> String { + "Untitled".to_string() +} + +fn default_deleted_user() -> String { + "[deleted]".to_string() +} + +// Custom deserializer for selftext (empty strings should be None) +fn deserialize_selftext<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s: Option = Option::deserialize(deserializer)?; + Ok(s.filter(|text| !text.is_empty())) +} + +// Custom deserializer for thumbnail (filter out "self", "default", empty) +fn deserialize_thumbnail<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s: Option = Option::deserialize(deserializer)?; + Ok(s.filter(|thumb| { + !thumb.is_empty() && thumb != "self" && thumb != "default" && thumb != "nsfw" && thumb != "spoiler" + })) +} + +// Helper methods for extracting typed data from Things +impl RedditThing { + /// Try to get this Thing as a Post + pub fn as_post(&self) -> Option<&RedditPost> { + if self.kind != "t3" { + return None; + } + match &self.data { + RedditThingData::Post(post) => Some(post), + _ => None, + } + } + + /// Try to get this Thing as a Comment + pub fn as_comment(&self) -> Option<&RedditComment> { + if self.kind != "t1" { + return None; + } + match &self.data { + RedditThingData::Comment(comment) => Some(comment), + _ => None, + } + } + + /// Check if this is a "more comments" indicator + pub fn is_more_comments(&self) -> bool { + self.kind == "more" + } +} + +// Helper methods for working with listing responses +impl RedditListingResponse { + /// Extract all posts from this listing + pub fn get_posts(&self) -> Vec<&RedditPost> { + self.data.children.iter() + .filter_map(|thing| thing.as_post()) + .collect() + } + + /// Extract all comments from this listing + pub fn get_comments(&self) -> Vec<&RedditComment> { + self.data.children.iter() + .filter_map(|thing| thing.as_comment()) + .collect() + } + + /// Extract posts as owned values + pub fn into_posts(self) -> Vec { + self.data.children.into_iter() + .filter_map(|thing| { + if thing.kind == "t3" { + match thing.data { + RedditThingData::Post(post) => Some(post), + _ => None, + } + } else { + None + } + }) + .collect() + } + + /// Extract comments as owned values + pub fn into_comments(self) -> Vec { + self.data.children.into_iter() + .filter_map(|thing| { + if thing.kind == "t1" { + match thing.data { + RedditThingData::Comment(comment) => Some(comment), + _ => None, + } + } else { + None + } + }) + .collect() + } +} + + + Side B — contributor: tommy-mor Side B — commit message: [461bfaa7] finished plan Side B — unified diff (full patch): diff --git a/PLAN.md b/PLAN.md deleted file mode 100644 index bc7b3beecea4d63d723432b479a003967f910e45..0000000000000000000000000000000000000000 --- a/PLAN.md +++ /dev/null @@ -1,363 +0,0 @@ -# sorter2 storage & memory plan - -## Goal - -Fit a **decent chunk of Reddit** into a **256MB** Fly VM while keeping the product simple: one Rust binary, no external database service. - -**RAM should be bounded by query shape**, not dataset size — ideally one rank-centrality graph in memory at a time, plus runtime overhead. - -**`events.jsonl` remains the main database.** Everything on disk elsewhere is a **rebuildable projection**. - ---- - -## Architecture (target) - -``` - ┌─────────────────────────────────┐ - │ events.jsonl (source of truth) │ - └───────────────┬─────────────────┘ - │ - append on every mutation - │ - ▼ - ┌──────────────────────────────────────────────┐ - │ apply event → durable projection (on disk) │ - │ (same semantics as today's in-memory reducer) │ - └──────────────────────────────────────────────┘ - │ │ - ▼ ▼ - ┌──────────────────┐ ┌──────────────────────────┐ - │ entity_payloads │ │ reducer state per scope │ - │ (fat Reddit JSON)│ │ nodes, children, edges, … │ - └──────────────────┘ └──────────────────────────┘ - │ │ - └────────┬───────────┘ - ▼ - ┌──────────────────────────────────────────────┐ - │ RAM per request (or small LRU cache) │ - │ • one scope's GroupState for rank-centrality │ - │ • children + EntityData (small views) │ - │ • RC scratch allocations │ - │ → compute → render → drop / evict │ - └──────────────────────────────────────────────┘ -``` - -This is **event sourcing / CQRS**: - -| Layer | Role | -|-------|------| -| **JSONL** | Canonical write log; audit; disaster recovery | -| **Durable (RocksDB)** | Materialized read model + payload store; rebuildable from JSONL | -| **RAM** | One (or few) hot scopes for ranking and render | - -Durable is **not** a second source of truth. If projection and log diverge, **stream JSONL and rebuild durable**. - -The plan is sound, but only if the projection layer is treated as a crash-recoverable index: - -- JSONL append must mean "the bytes are recoverable after process or VM crash" (`flush` alone is not enough; use `sync_data` / fsync-equivalent on the append path or make an explicit weaker durability tradeoff). -- Durable projection writes are allowed to lag JSONL, but startup must catch up from the last applied event before serving reads. -- Event application must be exactly-once for the durable projection. Votes add edge weights, so accidentally replaying the same event twice changes rankings. -- Projection schema should stay boring and explicit: persist nodes, children, edge weights, voted pairs, and capped recent votes directly. Avoid clever nested abstractions if they make rebuilds, migrations, or audits harder. - ---- - -## What we have today (baseline) - -| Piece | Status | -|-------|--------| -| `events.jsonl` append-only log | ✓ source of truth | -| `EventLog::replay` streaming one event at a time | ✓ no full `Vec` at startup | -| `entity_store` / `entity_db` (RocksDB via `durable`) | ✓ fat payloads off-heap | -| `EntityData` in `GlobalTree` | ✓ small derived views in RAM | -| Full `GlobalTree` replayed at boot | ✗ all nodes, all scopes' `GroupState` in RAM | -| Rank-centrality | ✓ already scoped per parent; reads in-memory `GroupState` | - -**Measured RSS (release, ~395 imports, 2.8MB JSONL):** - -| Scenario | RSS | -|----------|-----| -| Empty data dir | ~14 MB (+ RocksDB baseline) | -| After boot with data | ~21–22 MB | -| Pre-offload (in-tree payloads + vec replay) | ~25–34 MB | - -Payload offload + streaming replay helped startup peak, but **the full in-memory reducer** is still the scaling ceiling. - ---- - -## What lives where (target) - -### JSONL (`events.jsonl`) - -All mutations, append-only: - -- `VoteRecorded` — scope, pair, ratios -- `EntityImported` — id, full upstream payload -- `NodeEnsured` — register path -- (legacy / other event types as present in log) - -### Durable / RocksDB (`{data_dir}/…`) - -Single embedded DB directory. Collections (names tentative): - -| Collection | Contents | Notes | -|------------|----------|-------| -| `entity_payloads` | `ItemId → JSON string` | **Done.** Fat Reddit API blobs | -| `nodes` | `ItemId → { data: EntityData, children: … }` | Small; no raw payload | -| `scopes/{parent}/…` | `GroupState` materialization | edges, voted_pairs, item_to_idx, recent_votes (capped) | - -Nested layout can follow durable's `Map → Map → Vec` patterns (see `durable/docs/001.md` Sorter sketch). - -### RAM - -| Resident | When | -|----------|------| -| Tokio, Axum, reqwest, RocksDB block cache (tuned) | always | -| **One scope slice** | per request (or LRU of few scopes, byte-capped) | -| Rank-centrality temporaries | during render for that scope | - -**Not** in RAM at steady state: all subreddits, all vote graphs, all payloads. - ---- - -## Write path - -Order matters: - -1. Append event to `events.jsonl` (must durably succeed first) -2. Apply event to durable projection (same logic as today's `apply_event`) -3. Invalidate / update in-memory scope cache if that scope is hot - -Journal worker already serializes votes disk → tree; extend to **disk → durable** instead of (eventually) **disk → full GlobalTree**. - -```rust -// conceptual -append(jsonl, event)?; -apply_to_durable(event)?; -scope_cache.invalidate(scope_for(event)); -``` - -On failure after (1): replay from log repairs projection on next boot or via `replay-index` command. - -Because JSONL and RocksDB cannot be committed atomically together, durable must record a projection cursor alongside the projection: - -- Prefer a monotonically increasing event sequence number in each JSONL event. -- Acceptable first version: byte offset + line checksum, as long as truncation and partial trailing lines are handled deliberately. -- Update projection data and cursor in the same RocksDB `WriteBatch`. -- On startup, read the cursor, scan only the JSONL tail after that cursor, apply missing events, then serve. -- If the cursor is missing, corrupt, or points past the log, rebuild durable from JSONL. - -This keeps the append-first rule simple: if the process dies after JSONL append but before RocksDB apply, catch-up repairs it; if it dies after RocksDB apply but before cursor update, the batch should not expose a cursor that skips work. - ---- - -## Read path - -For a page under parent scope `P` (e.g. `reddit.com/r/rust`): - -1. **Load scope** from durable (or scope LRU hit) - - `EntityData` + children for listing - - `GroupState` for ranking and pair selection -2. **Run rank-centrality** on that `GroupState` (requires RAM — that's fine) -3. **Render** -4. **Drop** scope from RAM or return to LRU - -Payload fetch (rare): `entity_store.get(id)` only when render needs fields not in `EntityData`. - ---- - -## Startup & recovery - -### Normal startup - -``` -open entity_db (RocksDB) -open / validate scope indexes in same DB -read projection cursor -stream only unapplied JSONL tail into durable -do NOT replay JSONL into RAM -serve requests (cold scopes loaded on demand) -``` - -### Rebuild projection - -``` -stream events.jsonl → apply_event → durable -(one line at a time; same as EventLog::replay today) -``` - -Run when: - -- First deploy of projection layer -- Detected corruption / missing durable dir -- Manual `replay-index` after restoring JSONL from backup - -JSONL is the only file you need to trust for recovery. - ---- - -## Scope cache (RAM bound) - -**Strict mode:** one scope in RAM at a time — simplest, lowest RAM. - -**Practical mode:** LRU cache with **byte budget** (e.g. 64–128MB for scopes on a 256MB VM): - -- Evict least-recently-used scope's `GroupState` + child views -- Reload from durable on next visit - -Eviction policy is independent of storage engine. - ---- - -## Rank-centrality - -No change to the algorithm. It already assumes a whole `GroupState` for one parent scope. - -Moving reducer to durable does **not** remove RC memory cost — it removes **holding every scope's graph at once**. - -Optional later: materialized score vectors on disk, invalidated on vote. Not required for v1 of this plan. - ---- - -## Durable mutations (future API) - -Separate **intent** from **apply** for batching and testability: - -```rust -let m = rankings.path().key("rust").key(day).push_end(score); -db.apply(m)?; // or batch.apply(&[m1, m2, m3]) -``` - -Benefits: - -- One RocksDB `WriteBatch` / one WAL flush per vote or import batch -- Serializable ops for tests -- Aligns with JSONL events at the app layer and storage ops at the durable layer - -Keep chained `entry().push()` as sugar over `path().…; apply()`. - -Type safety: use type-state path builders if we want compile-time nesting; erased `Vec` only if we accept runtime errors at apply. - ---- - -## Storage engine: RocksDB vs alternatives - -**Current choice:** RocksDB via vendored `durable/` workspace crate. - -| Engine | Verdict for sorter2 | -|--------|---------------------| -| **RocksDB** | Good default for LSM, prefix scans, write-heavy votes + bulk imports. C++ dep, tune block cache for 256MB. | -| **sled** | Pure Rust appeal; production reliability history gives pause. Not a priority switch. | -| **fjall** | Pure Rust LSM; evaluate with benchmarks if leaving RocksDB. | -| **redb** | Lighter pure Rust; fine for payload-only store; less ideal for heavy scattered writes across scopes. | -| **SQLite** | Wrong shape for nested fractal tree; OK for a single KV table only. | - -**Switching engines matters less than:** - -1. Batched durable writes -2. Not materializing full reducer in RAM -3. Scope-local load/evict - -RocksDB stays **narrow**: blob attic + materialized reducer projection. Not a replacement for JSONL. - ---- - -## Scaling story (before vs after) - -| | In-memory reducer (today) | Target (JSONL + durable projection) | -|--|---------------------------|-------------------------------------| -| **RAM grows with** | Total nodes + all scopes + (was) payloads | Hot scope count × scope size + runtime | -| **Disk grows with** | JSONL (+ entity_db payloads today) | JSONL + full durable projection | -| **Startup** | O(events) replay into RAM | O(1) open DB | -| **Fails when** | RSS > VM limit | Scope too large for one RC graph, or disk full | -| **Recovery** | Replay JSONL | Replay JSONL → rebuild durable | - -**Rough RAM per active subreddit (~500 posts, moderate votes):** - -| Component | Order of magnitude | -|-----------|-------------------| -| `EntityData` × 500 | 0.5–2 MB | -| `GroupState` | 1–5 MB | -| RC scratch | 1–5 MB | -| Runtime + tuned RocksDB | 15–25 MB | -| **Total one hot scope** | **~20–40 MB** | - -Multiple subreddits fit on 256MB with LRU eviction, not all resident at once. - ---- - -## Implementation phases - -### Phase 0 — Done - -- [x] Vend `durable` as workspace crate (`durable/`) -- [x] `entity_store`: payloads in `{data_dir}/entity_db` -- [x] Remove `entity_raw` from `NodeState` -- [x] `EventLog::replay`: stream JSONL, one `Event` at a time -- [x] `apply_event` in `state.rs` for replay semantics - -### Phase 1 — Durable projection (write path) - -- [ ] Single `Db` under `{data_dir}/store` (payloads + reducer) -- [ ] JSONL append path uses explicit durable sync semantics (`sync_data` / fsync-equivalent) or documents any weaker mode -- [ ] Add projection cursor metadata (event sequence, or byte offset + checksum) -- [ ] `apply_event` writes to durable collections (nodes, scopes) and cursor in one RocksDB `WriteBatch` -- [ ] Journal + Reddit import paths use same apply -- [ ] Batched writes where possible (one flush per vote / per import batch) -- [ ] Idempotency tests: replay/catch-up never double-applies vote edge weights -- [ ] Tests: apply event → read back from durable - -### Phase 2 — Stop full tree at boot - -- [ ] Startup: open durable, catch up from projection cursor, then serve -- [ ] No `GlobalTree::new()` + full replay into RAM on normal boot -- [ ] `replay-index` command / flag: stream JSONL → durable (offline rebuild) -- [ ] Recovery tests: crash after JSONL append, crash after projection write, corrupt/missing cursor -- [ ] Integration tests use replay-index fixture or temp DB - -### Phase 3 — Scope load on read - -- [ ] `ScopeView { parent, children, group }` loaded from durable -- [ ] HTML / vote / pair / fetch handlers take `ScopeView` instead of `&GlobalTree` -- [ ] Remove or shrink `Arc>` - -### Phase 4 — Scope cache - -- [ ] LRU with byte budget for hot scopes -- [ ] Invalidate on write to that scope -- [ ] Metrics: cache hit/miss, evictions, scope load time - -### Phase 5 — Durable API polish (optional) - -- [ ] Reified path/mutation API + `WriteBatch` integration in `durable` -- [ ] RocksDB tuning preset for 256MB Fly (`block_cache`, write buffers) -- [ ] Document `replay-index` in AGENTS.md - ---- - -## Non-goals (for now) - -- Distributed replication or multi-writer -- Replacing JSONL as canonical store -- SQL query layer over state -- Materialized rank scores on disk (unless RC latency forces it) -- Switching from RocksDB to sled without benchmarks - ---- - -## Open questions - -1. **One DB or two?** `{data_dir}/entity_db` today vs single `{data_dir}/store` — merge on Phase 1? -2. **Event identity** — add event sequence numbers now, or start with byte offset + checksum? -3. **Append durability mode** — pay fsync cost per mutation/batch, or document an acknowledged data-loss window? -4. **Scope key encoding** — string `ItemId` paths vs hashed; must match event `scope` field. -5. **Child scope wiring** — Reddit `apply_entity_under_parent` creates children without full path ensure; durable schema must preserve this. -6. **Clojure smoke tests** — still read `events.jsonl`; durable is internal. No change expected. -7. **Fly volume** — `/data` holds JSONL + RocksDB; monitor disk alongside RAM. - ---- - -## Summary - -**JSONL = write log. Durable = full reducer on disk + fat payloads. RAM = one rank-centrality graph (or a small LRU of scopes).** - -Option B from design discussions is not separate from “reducer in durable” — it **is** reducer in durable, with JSONL still owning writes and recovery. The incremental work is: projection on apply, drop full tree at boot, scope load on read, then cache with eviction.