Side A implements a complete, testable Reddit fetch worker (OAuth token handling, rate-limit/backoff logic, URL mapping, JSON parsing for posts/subreddits, and unit tests) that is real, functioning capability wired into AppState and routes. Side B is a smaller, valuable refactor—fixing duplicate `#entity-section` ids via a per-item selector and deduplicating vote-compare card rendering—but it's a cleanup/bugfix on top of existing UI rather than new lasting functionality.
constitution · epochs · watch · epoch 3
comparison
c_c124c217f89c (tommy-mor) vs c_c6beb77e8e71 (tommy-mor)
download prompt · raw event · cmp_d767f9f11d1382
council reasoning
A implements a full Reddit fetch pipeline (broker worker, OAuth, rate-limit/backoff, URL mapping, JSON parsers, tree updates, and browse/ensure wiring)—core lasting product capability. B mainly refactors UI targeting (id→data-entity-section) and reuses entity_section on the vote page with CSS cleanup; useful DRY/SSE polish but much smaller lasting design impact.
Side A replaces a Reddit import placeholder with a working background fetch system, including request queuing, OAuth support, rate-limit handling, caching/deduplication, JSON parsing, integration into application state, and automatic fetch triggering when Reddit items are viewed or added. Side B is primarily a UI refactor that introduces per-item entity-section selectors for DOM morphing and reuses the existing entity component in the vote page, improving maintainability but adding far less core functionality.
sides
A — c_c124c217f89c (tommy-mor)
message
[8d8230d1] reddit
diff preview
diff --git a/.gitignore b/.gitignore
index 4c7073f9fac0c30fd2050d79a60ef447af58ebeb..ada462e900d24a3a6d08165d158c80f79c35a5a5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@
data/
repomix-output.xml
dev-data/
+.env
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 7906a8547d56b8e6a48ef59c37aa82a8510fdee9..4677fedcb45292eebebe7e9cf6ce2f5738f18ddf 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -16,6 +16,7 @@ tower = "0.5"
tower-http = { version = "0.5", features = ["trace"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+reqwest = { version = "0.12", features = ["json"] }
[dev-dependencies]
reqwest = { version = "0.12", features = ["json"] }
diff --git a/server/src/html/mod.rs b/server/src/html/mod.rs
index 2864407ed6e8ec284a1dc663acf1805534566b62..df6505021d9f446c2b453e20e3eb3cf696a111f9 100644
--- a/server/src/html/mod.rs
+++ b/server/src/html/mod.rs
@@ -272,5 +272,16 @@ pub async fn home(State(state): State<AppState>, uri: Uri) -> impl IntoResponse
pub async fn browse(State(state): State<AppState>, uri: Uri) -> impl IntoResponse {
let item = ItemId::from_browse_uri(uri.path()).unwrap_or(ItemId::root());
+ if item.as_str().starts_with("reddit.com") {
+ let needs_fetch = {
+ let tree = state.tree.read().await;
+ tree.get(&item)
+ .map(|n| n.data.is_none())
+ .unwrap_or(true)
+ };
+ if needs_fetch {
+ state.reddit.request_fetch(item.clone());
+ }
+ }
item_page(state, uri, item).await
}
diff --git a/server/src/reddit.rs b/server/src/reddit.rs
index d203dca09245daf869b3aa942898447700ae69fb..90053ad03b1d7c8e94f325dd4ee64c2b4f7da900 100644
--- a/server/src/reddit.rs
+++ b/server/src/reddit.rs
@@ -1,4 +1,12 @@
-//! Reddit API import (async, decoupled from UI request path).
+//! Reddit API import via a single background worker (rate limits, dedup, backoff).
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use reqwest::{header, Client, StatusCode};
+use serde::Deserialize;
+use tokio::sync::{mpsc, RwLock};
use crate::{
path_types::ItemId,
@@ -10,12 +18,401 @@ pub fn ensure_partial_tree(tree: &mut GlobalTree, id: &ItemId) {
tree.ensure_path(id);
}
-/// Placeholder for Reddit JSON import. Returns entity data when implemented.
-pub async fn fetch_reddit_entity(_id: &ItemId) -> Option<EntityData> {
- None
+pub struct RedditCommand {
+ pub id: ItemId,
+}
+
+#[derive(Clone)]
+pub struct RedditBroker {
+ tx: mpsc::Sender<RedditCommand>,
+}
+
+#[derive(Clone)]
+struct RedditCredentials {
+ client_id: String,
+ client_secret: String,
+}
+
+struct OAuthToken {
+ access_token: String,
+ expires_at: Instant,
+}
+
+impl RedditBroker {
+ pub fn spawn(tree: Arc<RwLock<GlobalTree>>, user_agent: &str) -> Self {
+ let (tx, rx) = mpsc::channel(100);
+
+ let mut headers = header::HeaderMap::new();
+ headers.insert(
+ header::USER_AGENT,
+ header::HeaderValue::from_str(user_agent).expect("valid user agent"),
+ );
+
+ let client = Client::builder()
+ .default_headers(headers)
+ .timeout(Duration::from_secs(15))
+ .build()
+ .expect("reqwest client");
+
+ let creds = RedditCredentials::from_env();
+ tokio::spawn(reddit_worker(rx, tree, client, creds));
+
+ Self { tx }
+ }
+
+ /// Fire-and-forget: queue a fetch; worker updates the tree when done.
+ pub fn request_fetch(&self, id: ItemId) {
+ let _ = self.tx.try_send(RedditCommand { id });
+ }
+}
+
+impl RedditCredentials {
+ fn from_env() -> Option<Self> {
+ let client_id = std::env::var("REDDIT_CLIENT_ID").ok()?;
+ let client_secret = std::env::var("REDDIT_CLIENT_SECRET").ok()?;
+ if client_id.is_empty() || client_secret.is_empty() {
+ return None;
+ }
+ Some(Self {
+ client_id,
+ client_secret,
+ })
+ }
+}
+
+pub fn default_user_agent() -> String {
+ std::env::var("REDDIT_USER_AGENT").unwrap_or_else(|_| {
+ "web:sorter2.social:v0.0.1 (by /u/sorter2)".to_string()
+ })
}
-/// Apply fetched entity data to a node (called from async worker).
-pub fn apply_entity(tree: &mut GlobalTree, id: &ItemId, data: EntityData) {
- tree.set_entity_data(id, data);
+async fn reddit_worker(
+ mut rx: mpsc::Receiver<RedditCommand>,
+ tree: Arc<RwLock<GlobalTree>>,
+ client: Client,
+ creds: Option<RedditCredentials>,
+) {
+ let mut in_flight = HashSet::new();
+ let mut recently_fetched: HashMap<ItemId, Instant> = HashMap::new();
+ let mut current_delay = Duration::from_secs(1);
+ let mut oauth: Option<OAuthToken> = None;
+ let cache_ttl = Duration::from_secs(300);
+
+ while let Some(cmd) = rx.recv().await {
+ let now = Instant::now();
+ recently_fetched.retain(|_, t| now.duration_since(*t) < cache_ttl);
+
+ if in_flight.contains(&cmd.id) || recently_fetched.contains_key(&cmd.id) {
+ continue;
+ }
+
+ in_flight.insert(cmd.id.clone());
+ let fetch_id = cmd.id.clone();
+
+ tokio::time::sleep(current_delay).await;
+
+ if let Some(c) = &creds {
+ oauth = ensure_oauth_token(&client, c, oauth.take()).await;
+ }
+
+ let token = oauth.as_ref().map(|t| t.access_token.as_str());
+ let use_oauth = token.is_some();
+
+ match do_fetch(&client, &fetch_id, use_oauth, token).await {
+ Ok(FetchOutcome::Entity(data)) => {
+ let mut w = tree.write().await;
+ w.set_entity_data(&fetch_id, data);
+ recently_fetched.insert(fetch_id.clone(), Instant::now());
+ current_delay = Duration::from_millis(600);
+ }
+ Ok(FetchOutcome::NotFound) => {
+ recently_fetched.insert(fetch_id.clone(), Instant::now());
+ }
+ Ok(FetchOutcome::RateLimited { reset_secs }) => {
+ let wait = Duration::from_secs(reset_secs.max(1));
+ tracing::warn!(
+ "Reddit rate limit for {}; sleeping {}s",
+ fetch_id,
+ wait.as_secs()
+ );
+ tokio::time::sleep(wait).await;
+ current_delay = (current_delay * 2).min(Duration::from_secs(60));
+ }
+ Err(e) => {
+ tracing::warn!("Reddit fetch failed for {}: {}", fetch_id, e);
+ current_delay = (current_delay * 2).min(Duration::from_secs(60));
+ }
+ }
+
+ in_flight.remove(&fetch_id);
+ }
+}
+
+enum FetchOutcome {
+ Entity(EntityData),
+ NotFound,
+ RateLimited { reset_secs: u64 },
+}
+
+async fn ensure_oauth_token(
+ client: &Client,
+ creds: &RedditCredentials,
+ existing: Option<OAuthToken>,
+) -> Option<OAuthToken> {
+ if let Some(t) = existing {
+ if Instant::now() < t.expires_at - Duration::from_secs(60) {
+ return Some(t);
+ }
+ }
+
+ let resp = client
+ .post("https://www.reddit.com/api/v1/access_token")
+ .basic_auth(&creds.client_id, Some(&creds.client_secret))
+ .form(&[("grant_type", "client_credentials")])
+ .send()
+ .await;
+
+ let resp = match resp {
+ Ok(r) => r,
+ Err(e) => {
+ tracing::warn!("Reddit OAuth token request failed: {e}");
+ return None;
+ }
+ };
+
+ if !resp.status().is_success() {
+ tracing::warn!("Reddit OAuth token HTTP {}", resp.status());
+ return None;
+ }
+
+ #[derive(Deserialize)]
+ struct TokenResponse {
+ access_token: String,
+ expires_in: u64,
+ }
+
+ let body: TokenResponse = match resp.json().await {
+ Ok(b) => b,
+ Err(e) => {
+ tracing::warn!("Reddit OAuth token parse failed: {e}");
+ return None;
+ }
+ };
+
+ Some(OAuthToken {
+ access_token: body.access_token,
+ expires_at: Instant::now() + Duration::from_secs(body.expires_in),
+ })
+}
+
+async fn do_fetch(
+ client: &Client,
+ id: &ItemId,
+ use_oauth: bool,
+ bearer: Option<&str>,
+) -> Result<FetchOutcome, String> {
+ let url = map_item_to_reddit_api(id, use_oauth);
+ if url.is_empty() {
+ return Ok(FetchOutcome::NotFound);
+ }
+
+ let mut req = client.get(&url);
+ if let Some(token) = bearer {
+ req = req.bearer_auth(token);
+ }
+
+ let resp = req.send().await.map_err(|e| e.to_string())?;
+
+ if resp.status() == StatusCode::TOO_MANY_REQUESTS {
+ let reset = rate_limit_reset_secs(&resp);
+ return Ok(FetchOutcome::RateLimited { reset_secs: reset });
+ }
+
+ if resp.status() == StatusCode::SERVICE_UNAVAILABLE {
+ return Err("Reddit unavailable (503)".to_string());
+ }
+
+ if !resp.status().is_success() {
+ return Ok(FetchOutcome::NotFound);
+ }
+
+ if rate_limit_remaining(&resp) == Some(0) {
+ let reset = rate_limit_reset_secs(&resp);
+ return Ok(FetchOutcome::RateLimited { reset_secs: reset });
+ }
+
+ let bytes = resp.bytes().await.map_err(|e| e.to_string())?;
+ Ok(parse_reddit_json(id, &bytes)
+ .map(FetchOutcome::Entity)
+ .unwrap_or(FetchOutcome::NotFound))
+}
+
+fn rate_limit_remaining(resp: &reqwest::Response) -> Option<u64> {
+ resp.headers()
+ .get("x-ratelimit-remaining")
+ .and_then(|v| v.to_str().ok())
+ .and_then(|s| s.parse::<f64>().ok())
+ .map(|f| f.floor() as u64)
+}
+
+fn rate_limit_reset_secs(resp: &reqwest::Response) -> u64 {
+ resp.headers()
+ .get("x-ratelimit-reset")
+ .and_then(|v| v.to_str().ok())
+ .and_then(|s| s.parse::<f64>().ok())
+ .map(|f| f.ceil() as u64)
+ .unwrap_or(5)
+}
+
+/// Map canonical item id to Reddit JSON API URL.
+pub fn map_item_to_reddit_api(id: &ItemId, oauth: bool) -> String {
+ let path = id.as_str();
+ if !path.starts_with("reddit.com/") && path != "reddit.com" {
+ return String::new();
+ }
+
+ let base = if oauth {
+ "https://oauth.reddit.com"
+ } else {
+ "https://www.reddit.com"
+ };
+
+ let segments: Vec<&str> = path.split('/').collect();
+
+ if let Some(i) = segments.iter().position(|&p| p == "comments") {
+ if segments.len() > i + 1 {
+ let api_path = segments[1..=i + 1].join("/");
+ return format!("{base}/{api_path}.json?raw_json=1");
+ }
+ }
+
+ if segments.len() == 3 && segments[1] == "r" {
+ return format!("{base}/r/{}/about.json?raw_json=1", segments[2]);
+ }
+
+ String::new()
+}
+
+fn parse_reddit_json(id: &ItemId, bytes: &[u8]) -> Option<EntityData> {
+ let v: serde_json::Value = serde_json::from_slice(bytes).ok()?;
+ let segments: Vec<&str> = id.as_str().split('/').collect();
+
+ if segments.iter().any(|&p| p == "comments") {
+ parse_post_listing(&v)
+ } else {
+ parse_subreddit_about(&v)
+ }
+}
+
+fn parse_subreddit_about(v: &serde_json::Value) -> Option<EntityData> {
+ let data = v.get("data")?;
+ let title = data
+ .get("title")
+ .or_else(|| data.get("display_name"))
+ .and_then(|t| t.as_str())?
+ .to_string();
+ let body_html = data
+ .get("public_description_html")
+ .or_else(|| data.get("public_description"))
+ .and_then(|t| t.as_str())
+ .map(|s| s.to_string());
+ let thumb_url = data
+ .get("icon_img")
+ .or_else(|| data.get("community_icon"))
+ .and_then(|t| t.as_str())
+ .filter(|s| !s.is_empty())
+ .map(|s| s.to_string());
+
+ Some(EntityData {
+ title,
+ author: None,
+ body_html,
+ thumb_url,
+ })
+}
+
+fn parse_post_listing(v:
… preview truncated; 4,570 characters omittedB — c_c6beb77e8e71 (tommy-mor)
message
[8dab9b80] nice
diff preview
diff --git a/server/src/fetch/html.rs b/server/src/fetch/html.rs
index 3e0309ff1c2ac1b17923922d2340ba6710e0f9d1..dadf050515f0473943dad97df5d318032c8cb385 100644
--- a/server/src/fetch/html.rs
+++ b/server/src/fetch/html.rs
@@ -10,13 +10,18 @@ use crate::{
ui_action::UI_RPC_FIELD,
};
-fn entity_panel(node: &NodeState) -> Markup {
+/// CSS selector for Idiomorph / SSE updates of one entity block.
+pub fn entity_section_selector(item: &ItemId) -> String {
+ format!(r#"[data-entity-section="{}"]"#, item.as_str())
+}
+
+pub fn entity_panel(node: &NodeState) -> Markup {
if let Some(markup) = crate::render::reddit::entity_markup(node) {
return markup;
}
html! {
@if let Some(data) = &node.data {
- div id="entity-panel" class="entity-card" {
+ div class="entity-card" {
h2 { (data.title) }
@if let Some(author) = &data.author {
p class="muted small" { "by " (author) }
@@ -76,11 +81,11 @@ pub fn fetch_entity_panel(item: &ItemId, has_data: bool, fetching: bool) -> Mark
}
}
-/// Entity card + fetch control (target `#entity-section` for Idiomorph / SSE).
+/// Entity card + fetch control (morph target [`entity_section_selector`]).
pub fn entity_section(item: &ItemId, node: &NodeState, fetching: bool) -> Markup {
let has_data = node.data.is_some();
html! {
- section id="entity-section" class="demo-panel" {
+ section class="entity-section demo-panel" data-entity-section=(item.as_str()) {
(entity_panel(node))
(fetch_entity_panel(item, has_data, fetching))
}
diff --git a/server/src/fetch/mod.rs b/server/src/fetch/mod.rs
index 35f968c21c69ca557bab7951413e3cfbbccfebfd..f5759a34a1afe4069805441cefde6474a6403550 100644
--- a/server/src/fetch/mod.rs
+++ b/server/src/fetch/mod.rs
@@ -77,8 +77,9 @@ pub fn fetch_entity_stream(
let tree = state.tree.read().await;
let empty = NodeState::default();
let node = tree.get(&id).unwrap_or(&empty);
+ let sel = html::entity_section_selector(&id);
JsBuilder::new()
- .morph_selector("#entity-section", html::entity_section(&id, node, true))
+ .morph_selector(&sel, html::entity_section(&id, node, true))
.build()
};
yield Ok(js_event(fetching_js));
@@ -100,12 +101,13 @@ pub fn fetch_entity_stream(
FetchJobResult::Imported(_)
| FetchJobResult::NotFound
| FetchJobResult::SkippedCached
- | FetchJobResult::SkippedDuplicate => {
+ | FetchJobResult::SkippedDuplicate => {
let tree = state.tree.read().await;
let empty = NodeState::default();
let node = tree.get(&id).unwrap_or(&empty);
+ let sel = html::entity_section_selector(&id);
let mut b = JsBuilder::new()
- .morph_selector("#entity-section", html::entity_section(&id, node, false));
+ .morph_selector(&sel, html::entity_section(&id, node, false));
if kind == FetchKind::Children {
b = b.morph_selector("#ranking-panel", ranking_panel(&id, node, &tree));
}
@@ -115,8 +117,9 @@ pub fn fetch_entity_stream(
let tree = state.tree.read().await;
let empty = NodeState::default();
let node = tree.get(&id).unwrap_or(&empty);
+ let sel = html::entity_section_selector(&id);
let js = JsBuilder::new()
- .morph_selector("#entity-section", html::entity_section(&id, node, false))
+ .morph_selector(&sel, html::entity_section(&id, node, false))
.raw(&error_js(&format!("Reddit rate limit — retry in {reset_secs}s.")))
.build();
yield Ok(js_event(js));
@@ -125,8 +128,9 @@ pub fn fetch_entity_stream(
let tree = state.tree.read().await;
let empty = NodeState::default();
let node = tree.get(&id).unwrap_or(&empty);
+ let sel = html::entity_section_selector(&id);
let js = JsBuilder::new()
- .morph_selector("#entity-section", html::entity_section(&id, node, false))
+ .morph_selector(&sel, html::entity_section(&id, node, false))
.raw(&error_js(&format!("Fetch failed: {msg}")))
.build();
yield Ok(js_event(js));
diff --git a/server/src/html/vote.rs b/server/src/html/vote.rs
index 89ed621ad861214e147add2062224fc4137ff8ad..48752f4d78d4762d1badc44e8df008bf5a45bab4 100644
--- a/server/src/html/vote.rs
+++ b/server/src/html/vote.rs
@@ -8,6 +8,7 @@ use maud::{html, Markup};
use serde::Deserialize;
use crate::{
+ fetch::html::entity_section,
form_template::template_json_compact,
html::JsBuilder,
pair::{children_of, resolve_pair, suggest_next_pair_in_pool},
@@ -169,37 +170,13 @@ pub(crate) fn vote_recorded_morph(
}
fn vote_compare_item_card(tree: &GlobalTree, item: &ItemId, side_class: &str) -> Markup {
- let href = item_href(item);
- let title = child_title(tree, item);
+ let node = tree.get(item).cloned().unwrap_or_else(|| NodeState {
+ id: item.clone(),
+ ..Default::default()
+ });
html! {
div class=(format!("vote-compare-side {side_class}")) {
- a class=(format!("vote-compare-item {side_class}")) href=(href) {
- @if let Some(row) = crate::render::reddit::child_row_markup(tree, item, &href) {
- (row)
- } @else {
- strong { (title) }
- }
- }
- @if let Some(node) = tree.get(item) {
- @if crate::render::reddit::is_reddit_post(item) {
- @if let Some(data) = &node.data {
- @if let Some(src) = data.image_url.as_ref().or(data.thumb_url.as_ref()) {
- figure class="vote-compare-figure" {
- img class="vote-compare-image" src=(src) alt="" loading="lazy";
- }
- }
- @if let Some(author) = &data.author {
- p class="muted small" { "by " (author) }
- }
- }
- } @else if let Some(data) = &node.data {
- @if let Some(body) = &data.body_html {
- div class="vote-compare-item-body" {
- (maud::PreEscaped(body))
- }
- }
- }
- }
+ (entity_section(item, &node, false))
}
}
}
@@ -270,9 +247,6 @@ pub async fn vote_page(
(vote_compare_item_card(&tree, &right, "vote-compare-right"))
}
(vote_back_nav(&parent))
- div id="vote-edge-history-region" {
- (edge_history)
- }
form id="vote-compare-form" method="POST" action="/ui" {
input type="hidden" name=(UI_RPC_FIELD) value=(rpc_json);
input type="hidden" name="ratio_left" id="vote-ratio-left" value="50";
@@ -285,6 +259,9 @@ pub async fn vote_page(
}
(vote_compare_actions(&parent, next_pair.as_ref()))
}
+ div id="vote-edge-history-region" {
+ (edge_history)
+ }
}
};
diff --git a/server/src/render/reddit.rs b/server/src/render/reddit.rs
index 4a18de93cf57d8395ded3caa373a708f39b3f43f..c4f98fc760c32ce1b41a91bd41e97908bd198937 100644
--- a/server/src/render/reddit.rs
+++ b/server/src/render/reddit.rs
@@ -11,7 +11,7 @@ pub fn is_reddit_post(id: &ItemId) -> bool {
id.as_str().starts_with("reddit.com/") && id.as_str().contains("/comments/")
}
-/// Post detail card (`#entity-panel`).
+/// Post detail card (inside [`crate::fetch::html::entity_panel`]).
pub fn entity_markup(node: &NodeState) -> Option<Markup> {
if !is_reddit_post(&node.id) {
return None;
@@ -41,7 +41,7 @@ pub fn child_row_markup(tree: &GlobalTree, id: &ItemId, href: &str) -> Option<Ma
fn post_entity_card(data: &EntityData) -> Markup {
let image = data.image_url.as_ref().or(data.thumb_url.as_ref());
html! {
- div id="entity-panel" class="entity-card reddit-post" {
+ div class="entity-card reddit-post" {
h2 { (data.title) }
@if let Some(author) = &data.author {
p class="muted small" { "by " (author) }
diff --git a/server/static/sorter.css b/server/static/sorter.css
index b95718ce463b200a5667d3014dac2119836a0bef..9379c5f41dedbd41fd8951099a118de1da2a62f4 100644
--- a/server/static/sorter.css
+++ b/server/static/sorter.css
@@ -254,38 +254,11 @@ h1 {
}
.vote-compare-side {
- background: var(--panel);
- border: 1px solid var(--border);
- border-radius: 8px;
- padding: 1rem;
min-height: 120px;
}
-.vote-compare-item {
- color: var(--accent);
- text-decoration: none;
- display: block;
-}
-
-.vote-compare-item:hover {
- text-decoration: underline;
-}
-
-.vote-compare-figure {
- margin: 0.75rem 0 0;
-}
-
-.vote-compare-image {
- display: block;
- max-width: 100%;
- height: auto;
- border-radius: 6px;
- border: 1px solid var(--border);
-}
-
-.vote-compare-item-body {
- margin-top: 0.75rem;
- font-size: 0.9rem;
+.vote-compare-side .entity-section {
+ margin: 0;
}
.vote-compare-nav {
Hardlinks — judgments / attempts / prompt
judgments
attempts
Prompt text is loaded only by the download route.