{"messages":[{"content":"You are a constitutional council ranking individual git commits for ownership allocation.\n\nCompare these two commits. Decide which contributed more lasting value to the project.\n\nJudge substance, not spectacle:\n- Prefer correct, lasting design and real bugfixes over churn, formatting, renames, or generated noise.\n- Prefer clarity and necessity over sheer line count. A small precise change can beat a large diffuse one.\n- Do not favor a side merely because its patch is longer or noisier.\n- Weight what the change does for the project, not the contributor's name.\n\nReturn ONLY a JSON object: {\"winner\": \"A\" or \"B\", \"ratio\": \"N:M\", \"explanation\": \"...\"}\nThe explanation must cite concrete differences in the patches (1-3 sentences).\n\nSide A — contributor: tommy-mor\nSide A — commit message:\n[3f35edab] progress\n\nSide A — unified diff (full patch):\ndiff --git a/server/src/api/mod.rs b/server/src/api/mod.rs\nindex a10ce662105cff8fad949c6b83f7035ce79bed18..a986f706ea4b261cbaf004c02b4cf84184b41371 100644\n--- a/server/src/api/mod.rs\n+++ b/server/src/api/mod.rs\n@@ -3,6 +3,7 @@ mod helpers;\n mod rpc;\n mod stream;\n mod validate;\n+mod ui_html;\n mod web_post;\n \n pub use auth::{\n@@ -33,6 +34,7 @@ pub use stream::{get_html_stream, get_stream};\n \n pub use validate::{normalize_room_and_thread, validate_ingest_document, ValidatedIngest};\n \n+pub use ui_html::post_ui_html;\n pub use web_post::{check_web_ingest, post_web_ingest, post_web_redact};\n \n #[cfg(test)]\ndiff --git a/server/src/api/ui_html.rs b/server/src/api/ui_html.rs\nnew file mode 100644\nindex 0000000000000000000000000000000000000000..2b40a72059981d558768f73d189b991f3448c257\n--- /dev/null\n+++ b/server/src/api/ui_html.rs\n@@ -0,0 +1,139 @@\n+//! Single `POST /ui` entry for browser [`crate::html::ui_action::HtmlUiAction`] (JSON in `__rpc__` + holes).\n+\n+use axum::{\n+ body::Body,\n+ extract::State,\n+ http::{header, HeaderMap, StatusCode},\n+ response::{IntoResponse, Response},\n+ Form,\n+};\n+use axum_extra::extract::cookie::CookieJar;\n+use std::collections::HashMap;\n+\n+use crate::{\n+ api::{\n+ auth::optional_principal,\n+ web_post::{run_check_web_ingest, run_post_web_ingest, run_post_web_redact, WebPostForm, WebRedactForm},\n+ },\n+ html::{\n+ fragment_public_new_thread_form, fragment_room_new_thread_form, login_to_post_hint_markup,\n+ parse_html_ui_from_form, user_can_post_room, user_can_view_room, HtmlUiAction, JsBuilder,\n+ ThreadNav,\n+ },\n+ state::AppState,\n+};\n+\n+pub async fn post_ui_html(\n+ State(state): State,\n+ headers: HeaderMap,\n+ jar: CookieJar,\n+ Form(form): Form>,\n+) -> impl IntoResponse {\n+ let action = match parse_html_ui_from_form(&form) {\n+ Ok(a) => a,\n+ Err(e) => return ui_js_warn(&e.to_string()).into_response(),\n+ };\n+\n+ match action {\n+ HtmlUiAction::PostIngest {\n+ room,\n+ thread_tag,\n+ text,\n+ error_target,\n+ form_id,\n+ } => {\n+ run_post_web_ingest(\n+ &state,\n+ &headers,\n+ &jar,\n+ WebPostForm {\n+ room,\n+ thread_tag,\n+ text,\n+ error_target,\n+ form_id,\n+ },\n+ )\n+ .await\n+ }\n+ HtmlUiAction::CheckIngest {\n+ room,\n+ thread_tag,\n+ text,\n+ error_target,\n+ form_id,\n+ } => {\n+ run_check_web_ingest(\n+ &state,\n+ &headers,\n+ &jar,\n+ WebPostForm {\n+ room,\n+ thread_tag,\n+ text,\n+ error_target,\n+ form_id,\n+ },\n+ )\n+ .await\n+ }\n+ HtmlUiAction::RedactPost { post_id } => {\n+ run_post_web_redact(&state, &headers, &jar, WebRedactForm { post_id }).await\n+ }\n+ HtmlUiAction::ExpandPublicNewThreadForm => {\n+ let reduced = state.reduced.read().await;\n+ let user = optional_principal(&headers, &jar, &reduced);\n+ drop(reduced);\n+ let markup = if user.is_some() {\n+ fragment_public_new_thread_form(true)\n+ } else {\n+ login_to_post_hint_markup()\n+ };\n+ JsBuilder::new()\n+ .morph_selector(\"#public-new-thread-ui-slot\", markup)\n+ .into_response()\n+ }\n+ HtmlUiAction::ExpandRoomNewThreadForm { room_wire } => {\n+ let room_wire = room_wire.trim().to_string();\n+ if room_wire.is_empty() {\n+ return ui_js_warn(\"missing room\").into_response();\n+ }\n+ let reduced = state.reduced.read().await;\n+ let user = optional_principal(&headers, &jar, &reduced);\n+ if !reduced.rooms.contains(&room_wire) {\n+ drop(reduced);\n+ return ui_js_warn(\"room not found\").into_response();\n+ }\n+ if !user_can_view_room(&reduced, &room_wire, user.as_deref()) {\n+ drop(reduced);\n+ return ui_js_warn(\"forbidden\").into_response();\n+ }\n+ let can_post = user\n+ .as_ref()\n+ .map(|u| user_can_post_room(&reduced, &room_wire, u))\n+ .unwrap_or(false);\n+ drop(reduced);\n+ let Some(nav) = ThreadNav::from_room_id(&room_wire) else {\n+ return ui_js_warn(\"bad room\").into_response();\n+ };\n+ let markup = if can_post {\n+ fragment_room_new_thread_form(&nav, true)\n+ } else {\n+ login_to_post_hint_markup()\n+ };\n+ JsBuilder::new()\n+ .morph_selector(\"#room-new-thread-ui-slot\", markup)\n+ .into_response()\n+ }\n+ }\n+}\n+\n+fn ui_js_warn(msg: &str) -> Response {\n+ use crate::html::js_string_literal;\n+ let js = format!(\"console.warn({});\", js_string_literal(msg));\n+ Response::builder()\n+ .status(StatusCode::OK)\n+ .header(header::CONTENT_TYPE, \"text/javascript; charset=utf-8\")\n+ .body(Body::from(js))\n+ .unwrap()\n+}\ndiff --git a/server/src/api/web_post.rs b/server/src/api/web_post.rs\nindex a64010e382d3039821c836a5529adad0fe67cce5..265025f41ff1548d05b2d2d5d84202245388053f 100644\n--- a/server/src/api/web_post.rs\n+++ b/server/src/api/web_post.rs\n@@ -222,8 +222,18 @@ pub async fn post_web_redact(\n jar: CookieJar,\n Form(form): Form,\n ) -> impl IntoResponse {\n+ run_post_web_redact(&state, &headers, &jar, form).await\n+}\n+\n+/// Shared with [`crate::api::ui_html::post_ui_html`].\n+pub(crate) async fn run_post_web_redact(\n+ state: &AppState,\n+ headers: &HeaderMap,\n+ jar: &CookieJar,\n+ form: WebRedactForm,\n+) -> Response {\n let reduced = state.reduced.read().await;\n- let Some(_username) = optional_principal(&headers, &jar, &reduced) else {\n+ let Some(_username) = optional_principal(headers, jar, &reduced) else {\n drop(reduced);\n return js_redirect(\"/login\").into_response();\n };\n@@ -239,8 +249,8 @@ pub async fn post_web_redact(\n return js_redirect(\"/login\").into_response();\n };\n \n- match rpc_post_redact(&state, &headers, form.post_id).await {\n- Ok(RpcResult::RedactPostOk {}) => redact_success_response(&state).await.into_response(),\n+ match rpc_post_redact(state, headers, form.post_id).await {\n+ Ok(RpcResult::RedactPostOk {}) => redact_success_response(state).await.into_response(),\n Ok(_) => (StatusCode::BAD_REQUEST, \"unexpected response\").into_response(),\n Err((msg, hint)) => {\n let detail = hint.as_deref().unwrap_or(\"\");\n@@ -255,8 +265,18 @@ pub async fn post_web_ingest(\n jar: CookieJar,\n Form(form): Form,\n ) -> impl IntoResponse {\n+ run_post_web_ingest(&state, &headers, &jar, form).await\n+}\n+\n+/// Shared with [`crate::api::ui_html::post_ui_html`] (`POST /ui`).\n+pub(crate) async fn run_post_web_ingest(\n+ state: &AppState,\n+ headers: &HeaderMap,\n+ jar: &CookieJar,\n+ form: WebPostForm,\n+) -> Response {\n let reduced = state.reduced.read().await;\n- let Some(_username) = optional_principal(&headers, &jar, &reduced) else {\n+ let Some(_username) = optional_principal(headers, jar, &reduced) else {\n drop(reduced);\n return js_redirect(\"/login\").into_response();\n };\n@@ -282,8 +302,8 @@ pub async fn post_web_ingest(\n .into_response();\n }\n \n- match rpc_post_with_bearer(&state, &bearer, room.clone(), thread_tag.clone(), text).await {\n- Ok(RpcResult::PostOk { .. }) => post_success_response(&state, &form, &headers, &jar)\n+ match rpc_post_with_bearer(state, &bearer, room.clone(), thread_tag.clone(), text).await {\n+ Ok(RpcResult::PostOk { .. }) => post_success_response(state, &form, headers, jar)\n .await\n .into_response(),\n Ok(_) => form_js_error(&form, \"unexpected response\", \"Post did not return PostOk.\").into_response(),\n@@ -297,8 +317,18 @@ pub async fn check_web_ingest(\n jar: CookieJar,\n Form(form): Form,\n ) -> impl IntoResponse {\n+ run_check_web_ingest(&state, &headers, &jar, form).await\n+}\n+\n+/// Shared with [`crate::api::ui_html::post_ui_html`] (`POST /ui`).\n+pub(crate) async fn run_check_web_ingest(\n+ state: &AppState,\n+ headers: &HeaderMap,\n+ jar: &CookieJar,\n+ form: WebPostForm,\n+) -> Response {\n let reduced = state.reduced.read().await;\n- let Some(_username) = optional_principal(&headers, &jar, &reduced) else {\n+ let Some(_username) = optional_principal(headers, jar, &reduced) else {\n drop(reduced);\n return js_redirect(\"/login\").into_response();\n };\n@@ -324,7 +354,7 @@ pub async fn check_web_ingest(\n return js_clear_errors(&form_error_target(&form)).into_response();\n }\n \n- match rpc_check_with_bearer(&state, &bearer, room, form.text.clone()).await {\n+ match rpc_check_with_bearer(state, &bearer, room, form.text.clone()).await {\n Ok(RpcResult::CheckOk { .. }) => js_clear_errors(&form_error_target(&form)).into_response(),\n Ok(_) => form_js_error(&form, \"unexpected response\", \"Check did not return CheckOk.\").into_response(),\n Err((msg, hint)) => form_js_error(&form, &msg, hint.as_deref().unwrap_or(\"\")).into_response(),\ndiff --git a/server/src/form_template.rs b/server/src/form_template.rs\nnew file mode 100644\nindex 0000000000000000000000000000000000000000..3709c2c09a859da006e4af173413d5d235bc19be\n--- /dev/null\n+++ b/server/src/form_template.rs\n@@ -0,0 +1,142 @@\n+//! Plan2-style JSON templates with `{\"$form\": \"field_name\"}` holes, filled from\n+//! `application/x-www-form-urlencoded` (or any `String` → `String` map) **before**\n+//! deserializing into a typed struct.\n+//!\n+//! # Wire format\n+//!\n+//! Templates are **compact JSON** (`serde_json::to_string`): one line, no pretty\n+//! printing, strings escaped per JSON rules (`\\\"`, `\\n`, etc.). Embed that string\n+//! in HTML attributes or text nodes with normal HTML escaping (e.g. maud), not\n+//! bespoke encodings.\n+//!\n+//! # Power vs flat hidden fields\n+//!\n+//! A form is always a string→string map. You can fake depth with dotted keys (`a.b.c`),\n+//! but one structured blob (`__rpc__` = compact JSON) gives you nested objects,\n+//! arrays, and optional fields without inventing a new naming scheme each time.\n+//!\n+//! # Security\n+//!\n+//! Substitution runs **before** `serde` into your command type. It does not fix\n+//! authorization: if the client can replace the hidden `__rpc__` value, they can\n+//! change the command shape unless you validate (signed blob, server-side session\n+//! context, or treat the blob as hints only). Same threat model as any hidden field.\n+\n+use serde::Serialize;\n+use serde_json::Value;\n+use std::collections::HashMap;\n+\n+/// Serialize a value to compact JSON for a hidden `__rpc__` (or similar) field.\n+pub fn template_json_compact(v: &T) -> serde_json::Result {\n+ serde_json::to_string(v)\n+}\n+\n+/// Recursively walk the JSON AST and replace `{\"$form\": \"key\"}` with the submitted\n+/// string for `key` (empty if missing). Other keys are unchanged.\n+pub fn substitute_form_vars(val: &mut Value, form_data: &HashMap) {\n+ match val {\n+ Value::Object(map) => {\n+ if map.len() == 1 {\n+ if let Some(Value::String(field_name)) = map.get(\"$form\") {\n+ let submitted = form_data\n+ .get(field_name.as_str())\n+ .map(|s| s.as_str())\n+ .unwrap_or(\"\");\n+ *val = Value::String(submitted.to_string());\n+ return;\n+ }\n+ }\n+ for v in map.values_mut() {\n+ substitute_form_vars(v, form_data);\n+ }\n+ }\n+ Value::Array(arr) => {\n+ for v in arr.iter_mut() {\n+ substitute_form_vars(v, form_data);\n+ }\n+ }\n+ _ => {}\n+ }\n+}\n+\n+/// Parse JSON, apply [`substitute_form_vars`], return the mutated value.\n+pub fn fill_template_from_form(\n+ template_json: &str,\n+ form_data: &HashMap,\n+) -> Result {\n+ let mut v: Value = serde_json::from_str(template_json)?;\n+ substitute_form_vars(&mut v, form_data);\n+ Ok(v)\n+}\n+\n+#[cfg(test)]\n+mod tests {\n+ use super::*;\n+ use serde::Deserialize;\n+\n+ #[derive(Debug, Deserialize, PartialEq, Eq)]\n+ struct Demo {\n+ room: String,\n+ thread_tag: String,\n+ nested: Nested,\n+ }\n+\n+ #[derive(Debug, Deserialize, PartialEq, Eq)]\n+ struct Nested {\n+ text: String,\n+ }\n+\n+ #[test]\n+ fn holes_become_strings() {\n+ let json = r#\"{\n+ \"room\": \"public\",\n+ \"thread_tag\": {\"$form\": \"tag\"},\n+ \"nested\": {\"text\": {\"$form\": \"body\"}}\n+ }\"#;\n+ let mut form = HashMap::new();\n+ form.insert(\"tag\".into(), \"foo\".into());\n+ form.insert(\"body\".into(), \"hello\\nworld\".into());\n+\n+ let v = fill_template_from_form(json, &form).unwrap();\n+ let d: Demo = serde_json::from_value(v).unwrap();\n+ assert_eq!(\n+ d,\n+ Demo {\n+ room: \"public\".into(),\n+ thread_tag: \"foo\".into(),\n+ nested: Nested {\n+ text: \"hello\\nworld\".into(),\n+ },\n+ }\n+ );\n+ }\n+\n+ #[test]\n+ fn missing_form_key_is_empty_string() {\n+ let json = r#\"{\"x\": {\"$form\": \"nope\"}}\"#;\n+ let mut form = HashMap::new();\n+ form.insert(\"other\".into(), \"y\".into());\n+ let v = fill_template_from_form(json, &form).unwrap();\n+ assert_eq!(v[\"x\"], \"\");\n+ }\n+\n+ #[test]\n+ fn array_of_holes() {\n+ let json = r#\"{\"items\": [{\"$form\": \"a\"}, {\"$form\": \"b\"}]}\"#;\n+ let mut form = HashMap::new();\n+ form.insert(\"a\".into(), \"1\".into());\n+ form.insert(\"b\".into(), \"2\".into());\n+ let v = fill_template_from_form(json, &form).unwrap();\n+ assert_eq!(v[\"items\"], serde_json::json!([\"1\", \"2\"]));\n+ }\n+\n+ #[test]\n+ fn template_json_compact_escapes_and_single_line() {\n+ let s = template_json_compact(&serde_json::json!({\n+ \"x\": \"quote\\\"and\\nnewline\"\n+ }))\n+ .unwrap();\n+ assert!(!s.contains('\\n'));\n+ assert!(s.contains(\"\\\\\\\"\") || s.contains(\"\\\\n\"));\n+ }\n+}\ndiff --git a/server/src/html/forum.rs b/server/src/html/forum.rs\nindex f789e347dc136f8ffd356f4492f0bd76cea04bf0..b1c037f3dd93c7bb96d6624cab6019228432c501 100644\n--- a/server/src/html/forum.rs\n+++ b/server/src/html/forum.rs\n@@ -11,12 +11,15 @@ use crate::{\n api::optional_principal,\n canonical_path::{canonicalize_item, canonicalize_tag},\n events::ThreadCapability,\n+ form_template::template_json_compact,\n identity::parse_username,\n reducer::{scope_from_room_wire, ReducerState, ScopeId},\n state::AppState,\n timeago,\n };\n \n+use super::ui_action::{HtmlUiAction, UI_RPC_FIELD};\n+\n use super::{\n bc_segment, bc_threads, cli_panel, layout, now_ms, profile_href, recency_class,\n render_linkified_with_embeds_in_scope, theme_from_jar, theme_next_from_uri, JsBuilder,\n@@ -289,7 +292,7 @@ fn rooms_for_user(reduced: &ReducerState, username: &str) -> Vec {\n v\n }\n \n-fn user_can_view_room(reduced: &ReducerState, room_id: &str, username: Option<&str>) -> bool {\n+pub(crate) fn user_can_view_room(reduced: &ReducerState, room_id: &str, username: Option<&str>) -> bool {\n if !reduced.rooms.contains(room_id) {\n return false;\n }\n@@ -299,7 +302,7 @@ fn user_can_view_room(reduced: &ReducerState, room_id: &str, username: Option<&s\n reduced.user_has_cap(room_id, u, ThreadCapability::View)\n }\n \n-fn user_can_post_room(reduced: &ReducerState, room_id: &str, username: &str) -> bool {\n+pub(crate) fn user_can_post_room(reduced: &ReducerState, room_id: &str, username: &str) -> bool {\n reduced.user_has_cap(room_id, username, ThreadCapability::Post)\n }\n \n@@ -591,7 +594,6 @@ pub async fn home(\n let nav = ThreadNav::public();\n let reduced_read = state.reduced.read().await;\n let strip = auth_strip(&headers, &jar, &reduced_read);\n- let show_forms = user.is_some();\n drop(reduced_read);\n \n let page = layout(\n@@ -617,8 +619,14 @@ pub async fn home(\n }\n }\n p class=\"muted\" { \"dark = time-ordered · light = vote-ranked\" }\n+ div class=\"thread-feed-toolbar\" {\n+ form method=\"POST\" action=\"/ui\" {\n+ input type=\"hidden\" name=(UI_RPC_FIELD) value=(expand_public_new_thread_rpc_value());\n+ button type=\"submit\" class=\"section-add-btn\" { \"+\" }\n+ }\n+ }\n+ div id=\"public-new-thread-ui-slot\" {}\n (render_thread_feed(Some(&nav), \"thread-feed\", &public_rows, now))\n- (new_thread_form_public(show_forms))\n (cli_panel(\"npx slugsocial public forum list\"))\n },\n None,\n@@ -901,7 +909,13 @@ pub async fn room_page(\n h3 { \"threads\" }\n (render_thread_feed(Some(&nav), \"room-thread-feed\", &rows, now))\n @if show_new {\n- (new_thread_form_for_room(&nav, show_new))\n+ div class=\"thread-feed-toolbar\" {\n+ form method=\"POST\" action=\"/ui\" {\n+ input type=\"hidden\" name=(UI_RPC_FIELD) value=(expand_room_new_thread_rpc_value(&nav));\n+ button type=\"submit\" class=\"section-add-btn\" { \"+\" }\n+ }\n+ }\n+ div id=\"room-new-thread-ui-slot\" {}\n }\n (cli_panel(&forum_cli))\n (cli_panel(&garden_cli))\n@@ -945,6 +959,31 @@ fn new_thread_form_for_room(nav: &ThreadNav, show: bool) -> Markup {\n }\n }\n \n+pub(crate) fn expand_public_new_thread_rpc_value() -> String {\n+ template_json_compact(&HtmlUiAction::ExpandPublicNewThreadForm).expect(\"static json\")\n+}\n+\n+pub(crate) fn expand_room_new_thread_rpc_value(nav: &ThreadNav) -> String {\n+ template_json_compact(&HtmlUiAction::ExpandRoomNewThreadForm {\n+ room_wire: nav.room_wire.clone(),\n+ })\n+ .expect(\"static json\")\n+}\n+\n+pub(crate) fn login_to_post_hint_markup() -> Markup {\n+ html! {\n+ p class=\"muted\" { \"log in to post\" }\n+ }\n+}\n+\n+pub(crate) fn fragment_public_new_thread_form(show: bool) -> Markup {\n+ new_thread_form_public(show)\n+}\n+\n+pub(crate) fn fragment_room_new_thread_form(nav: &ThreadNav, show: bool) -> Markup {\n+ new_thread_form_for_room(nav, show)\n+}\n+\n async fn thread_post_view_inner(\n state: AppState,\n tag: String,\ndiff --git a/server/src/html/mod.rs b/server/src/html/mod.rs\nindex 53041a0cf0b30e6f20749c92ecc15a4e9ac56e09..a4fd2dd60ac283f7eb9b2e64522501b21d1e27e9 100644\n--- a/server/src/html/mod.rs\n+++ b/server/src/html/mod.rs\n@@ -16,6 +16,7 @@ mod editor;\n mod forum;\n mod garden;\n mod search;\n+pub mod ui_action;\n use breadcrumb_path::OntologyPath;\n \n pub use auth::{auth_complete_page, auth_signed_in_fragment, choose_username_error_fragment, choose_username_page};\n@@ -27,9 +28,15 @@ pub use forum::{\n thread_post_collapse_deleted, thread_post_expand, thread_post_expand_deleted, thread_post_view,\n thread_view, ThreadNav,\n };\n+\n+pub(crate) use forum::{\n+ fragment_public_new_thread_form, fragment_room_new_thread_form, login_to_post_hint_markup,\n+ user_can_post_room, user_can_view_room,\n+};\n pub use garden::{garden_index, ontology_path, room_garden_index, room_ontology_path};\n pub use search::{search_page, search_results_fragment};\n pub use forum::user_profile_page;\n+pub use ui_action::{parse_html_ui_from_form, HtmlUiAction, HtmlUiParseError, UI_RPC_FIELD};\n \n /// Public profile URL path for a stored username (no `@`).\n pub(crate) fn profile_href(username: &str) -> String {\ndiff --git a/server/src/html/ui_action.rs b/server/src/html/ui_action.rs\nnew file mode 100644\nindex 0000000000000000000000000000000000000000..3bc69ecbb7e16a59d2d393b2d59f9cd3fedb12b1\n--- /dev/null\n+++ b/server/src/html/ui_action.rs\n@@ -0,0 +1,116 @@\n+//! Browser-only UI commands: JSON in hidden `__rpc__` plus hole fill ([`crate::form_template`]).\n+//! Not part of [`slug_types::RpcCommand`] (CLI / JSON API).\n+\n+use crate::form_template::fill_template_from_form;\n+use serde::{Deserialize, Serialize};\n+use serde_json::Value;\n+use std::collections::HashMap;\n+use thiserror::Error;\n+\n+/// Form field name for the compact JSON template (possibly with `{\"$form\":\"…\"}` holes).\n+pub const UI_RPC_FIELD: &str = \"__rpc__\";\n+\n+/// HTML form / fetch `POST /ui` payload after template fill and deserialization.\n+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]\n+#[serde(tag = \"action\", rename_all = \"snake_case\")]\n+pub enum HtmlUiAction {\n+ /// Same semantics as `POST /post` (forum ingest).\n+ PostIngest {\n+ room: String,\n+ thread_tag: String,\n+ text: String,\n+ #[serde(default)]\n+ error_target: Option,\n+ #[serde(default)]\n+ form_id: Option,\n+ },\n+ /// Same as `POST /post/check`.\n+ CheckIngest {\n+ room: String,\n+ thread_tag: String,\n+ text: String,\n+ #[serde(default)]\n+ error_target: Option,\n+ #[serde(default)]\n+ form_id: Option,\n+ },\n+ /// Same as `POST /post/redact`.\n+ RedactPost {\n+ post_id: String,\n+ },\n+ /// Morph `#public-new-thread-ui-slot` to the new-thread form (or login hint).\n+ ExpandPublicNewThreadForm,\n+ /// Morph `#room-new-thread-ui-slot` for the given room wire id.\n+ ExpandRoomNewThreadForm {\n+ room_wire: String,\n+ },\n+}\n+\n+#[derive(Debug, Error)]\n+pub enum HtmlUiParseError {\n+ #[error(\"missing __rpc__ field\")]\n+ MissingRpc,\n+ #[error(\"invalid template json: {0}\")]\n+ Template(serde_json::Error),\n+ #[error(\"invalid ui action: {0}\")]\n+ Action(serde_json::Error),\n+}\n+\n+/// Parse `__rpc__` JSON, apply `$form` holes from the rest of the form map, deserialize.\n+pub fn parse_html_ui_from_form(form: &HashMap) -> Result {\n+ let template = form\n+ .get(UI_RPC_FIELD)\n+ .ok_or(HtmlUiParseError::MissingRpc)?;\n+ let mut hole_map = form.clone();\n+ hole_map.remove(UI_RPC_FIELD);\n+ let v: Value = fill_template_from_form(template, &hole_map).map_err(HtmlUiParseError::Template)?;\n+ serde_json::from_value(v).map_err(HtmlUiParseError::Action)\n+}\n+\n+#[cfg(test)]\n+mod tests {\n+ use super::*;\n+\n+ #[test]\n+ fn round_trip_post_ingest_with_holes() {\n+ let template = serde_json::json!({\n+ \"action\": \"post_ingest\",\n+ \"room\": \"public\",\n+ \"thread_tag\": {\"$form\": \"thread_tag\"},\n+ \"text\": {\"$form\": \"text\"},\n+ \"error_target\": \"e\",\n+ \"form_id\": \"f\",\n+ });\n+ let mut form = HashMap::new();\n+ form.insert(\n+ UI_RPC_FIELD.to_string(),\n+ serde_json::to_string(&template).unwrap(),\n+ );\n+ form.insert(\"thread_tag\".into(), \"x\".into());\n+ form.insert(\"text\".into(), \"body\".into());\n+\n+ let a = parse_html_ui_from_form(&form).unwrap();\n+ assert_eq!(\n+ a,\n+ HtmlUiAction::PostIngest {\n+ room: \"public\".into(),\n+ thread_tag: \"x\".into(),\n+ text: \"body\".into(),\n+ error_target: Some(\"e\".into()),\n+ form_id: Some(\"f\".into()),\n+ }\n+ );\n+ }\n+\n+ #[test]\n+ fn expand_public_unit_variant() {\n+ let template = serde_json::json!({ \"action\": \"expand_public_new_thread_form\" });\n+ let mut form = HashMap::new();\n+ form.insert(\n+ UI_RPC_FIELD.to_string(),\n+ serde_json::to_string(&template).unwrap(),\n+ );\n+ let a = parse_html_ui_from_form(&form).unwrap();\n+ assert_eq!(a, HtmlUiAction::ExpandPublicNewThreadForm);\n+ }\n+}\ndiff --git a/server/src/lib.rs b/server/src/lib.rs\nindex b9d4b79791ba8dd3a3c26ff777943a2548092de5..5f5b6b0f01b29350c7415e5801ec5caa74f0b452 100644\n--- a/server/src/lib.rs\n+++ b/server/src/lib.rs\n@@ -3,6 +3,7 @@ pub mod paths;\n pub mod api;\n pub mod canonical_path;\n pub mod dsl;\n+pub mod form_template;\n pub mod html;\n pub mod event_log;\n pub mod events;\n@@ -33,6 +34,7 @@ pub fn create_app(state: AppState) -> Router {\n .route(\"/post\", post(api::post_web_ingest))\n .route(\"/post/redact\", post(api::post_web_redact))\n .route(\"/post/check\", post(api::check_web_ingest))\n+ .route(\"/ui\", post(api::post_ui_html))\n .route(\"/theme\", post(crate::html::post_theme))\n .route(\"/sse\", get(api::get_html_stream))\n .route(\"/stream\", get(api::get_stream))\n\n\nSide B — contributor: tommy-mor\nSide B — commit message:\n[6d04afc2] refactor\n\nSide B — unified diff (full patch):\ndiff --git a/Cargo.lock b/Cargo.lock\nindex 2cea973082716e761ef6f5dd5886acc08ff9aac0..8c43fb75c472b102e6e1d3b837dce3355be898f2 100644\n--- a/Cargo.lock\n+++ b/Cargo.lock\n@@ -17,6 +17,28 @@ version = \"1.0.102\"\n source = \"registry+https://github.com/rust-lang/crates.io-index\"\n checksum = \"7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c\"\n \n+[[package]]\n+name = \"async-stream\"\n+version = \"0.3.6\"\n+source = \"registry+https://github.com/rust-lang/crates.io-index\"\n+checksum = \"0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476\"\n+dependencies = [\n+ \"async-stream-impl\",\n+ \"futures-core\",\n+ \"pin-project-lite\",\n+]\n+\n+[[package]]\n+name = \"async-stream-impl\"\n+version = \"0.3.6\"\n+source = \"registry+https://github.com/rust-lang/crates.io-index\"\n+checksum = \"c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d\"\n+dependencies = [\n+ \"proc-macro2\",\n+ \"quote\",\n+ \"syn\",\n+]\n+\n [[package]]\n name = \"async-trait\"\n version = \"0.1.89\"\n@@ -1242,9 +1264,11 @@ dependencies = [\n name = \"sorter2-server\"\n version = \"0.0.1\"\n dependencies = [\n+ \"async-stream\",\n \"axum\",\n \"axum-extra\",\n \"dotenvy\",\n+ \"futures-util\",\n \"maud\",\n \"reqwest\",\n \"serde\",\ndiff --git a/server/Cargo.toml b/server/Cargo.toml\nindex bd600138b613bd0f546bdec217a5334cdcb20aa5..c940acb687fb141d21760a3d6656172013cf6f41 100644\n--- a/server/Cargo.toml\n+++ b/server/Cargo.toml\n@@ -18,6 +18,8 @@ tracing = \"0.1\"\n tracing-subscriber = { version = \"0.3\", features = [\"env-filter\"] }\n reqwest = { version = \"0.12\", features = [\"json\"] }\n dotenvy = \"0.15\"\n+async-stream = \"0.3\"\n+futures-util = { version = \"0.3\", default-features = false, features = [\"std\"] }\n \n [dev-dependencies]\n reqwest = { version = \"0.12\", features = [\"json\"] }\ndiff --git a/server/src/api/ui_html.rs b/server/src/api/ui_html.rs\nindex b33a84e8bb5e817b26592868d88090e6d664d950..7af6527d03c483f33f3469ce6766c01a554c5fe3 100644\n--- a/server/src/api/ui_html.rs\n+++ b/server/src/api/ui_html.rs\n@@ -6,7 +6,8 @@ use axum::{\n use std::collections::HashMap;\n \n use crate::{\n- html::{entity_section, input_panel, js_string_literal, ranking_panel, JsBuilder},\n+ fetch,\n+ html::{input_panel, js_string_literal, ranking_panel, JsBuilder},\n parser::parse_reddit_url,\n path_types::ItemId,\n reddit::ensure_partial_tree,\n@@ -89,18 +90,8 @@ pub async fn post_ui_html(\n },\n HtmlUiAction::FetchEntity { item } => {\n let id = parse_item_param(&item);\n- if id.is_root() {\n- return ui_js_warn(\"nothing to fetch for the root\").into_response();\n- }\n- state.queue_entity_fetch(id.clone());\n- let tree = state.tree.read().await;\n- let empty = crate::reducer::NodeState::default();\n- let node = tree.get(&id).unwrap_or(&empty);\n- let panel = entity_section(&id, node, true);\n- JsBuilder::new()\n- .morph_selector(\"#entity-section\", panel)\n- .into_response()\n- },\n+ fetch::fetch_entity_stream(state, id).into_response()\n+ }\n }\n }\n \ndiff --git a/server/src/fetch/html.rs b/server/src/fetch/html.rs\nnew file mode 100644\nindex 0000000000000000000000000000000000000000..63634508496e224c38b9ec0308b7a6086462f925\n--- /dev/null\n+++ b/server/src/fetch/html.rs\n@@ -0,0 +1,67 @@\n+//! Markup for entity import / “Fetch from Reddit” (`POST /ui`, SSE response).\n+\n+use maud::{html, Markup};\n+\n+use crate::{\n+ form_template::template_json_compact,\n+ path_types::ItemId,\n+ reddit::is_fetchable,\n+ reducer::NodeState,\n+ ui_action::UI_RPC_FIELD,\n+};\n+\n+fn entity_panel(node: &NodeState) -> Markup {\n+ html! {\n+ @if let Some(data) = &node.data {\n+ div id=\"entity-panel\" class=\"entity-card\" {\n+ h2 { (data.title) }\n+ @if let Some(author) = &data.author {\n+ p class=\"muted small\" { \"by \" (author) }\n+ }\n+ @if let Some(body) = &data.body_html {\n+ div class=\"entity-body\" { (maud::PreEscaped(body)) }\n+ }\n+ }\n+ }\n+ }\n+}\n+\n+/// Reddit/API import — `POST /ui` with `fetch_entity` returns an SSE stream.\n+pub fn fetch_entity_panel(item: &ItemId, has_data: bool, fetching: bool) -> Markup {\n+ if !is_fetchable(item) {\n+ return html! {};\n+ }\n+ let label = if fetching {\n+ \"Fetching…\"\n+ } else if has_data {\n+ \"Fetch more\"\n+ } else {\n+ \"Fetch from Reddit\"\n+ };\n+ let rpc = template_json_compact(&serde_json::json!({\n+ \"action\": \"fetch_entity\",\n+ \"item\": item.as_str(),\n+ }))\n+ .expect(\"fetch_entity rpc template\");\n+ html! {\n+ form method=\"post\" action=\"/ui\" id=\"fetch-entity-form\" class=\"fetch-entity-form\" {\n+ input type=\"hidden\" name=(UI_RPC_FIELD) value=(rpc);\n+ @if fetching {\n+ button type=\"submit\" class=\"btn-secondary\" disabled { (label) }\n+ } @else {\n+ button type=\"submit\" class=\"btn-secondary\" { (label) }\n+ }\n+ }\n+ }\n+}\n+\n+/// Entity card + fetch control (target `#entity-section` for Idiomorph / SSE).\n+pub fn entity_section(item: &ItemId, node: &NodeState, fetching: bool) -> Markup {\n+ let has_data = node.data.is_some();\n+ html! {\n+ section id=\"entity-section\" class=\"demo-panel\" {\n+ (entity_panel(node))\n+ (fetch_entity_panel(item, has_data, fetching))\n+ }\n+ }\n+}\ndiff --git a/server/src/fetch/mod.rs b/server/src/fetch/mod.rs\nnew file mode 100644\nindex 0000000000000000000000000000000000000000..2290f9d3a0f1cbf1806c6339f82a4515c11cc3d3\n--- /dev/null\n+++ b/server/src/fetch/mod.rs\n@@ -0,0 +1,115 @@\n+//! Entity import over `POST /ui` as SSE (Reddit worker in [`crate::reddit`]).\n+\n+pub mod html;\n+\n+use std::convert::Infallible;\n+use std::time::Duration;\n+\n+use async_stream::stream;\n+use axum::response::sse::{Event, KeepAlive, Sse};\n+use futures_util::Stream;\n+use serde::Serialize;\n+use tokio::sync::oneshot;\n+\n+use crate::{\n+ path_types::ItemId,\n+ reddit::FetchJobResult,\n+ reducer::NodeState,\n+ state::AppState,\n+};\n+\n+pub fn now_ms() -> i64 {\n+ let t = std::time::SystemTime::now()\n+ .duration_since(std::time::UNIX_EPOCH)\n+ .unwrap_or_default();\n+ t.as_millis() as i64\n+}\n+\n+#[derive(Serialize)]\n+struct SseMorphPayload {\n+ selector: &'static str,\n+ html: String,\n+}\n+\n+fn morph_complete_event(html: maud::Markup) -> Event {\n+ let payload = SseMorphPayload {\n+ selector: \"#entity-section\",\n+ html: html.into_string(),\n+ };\n+ let data = serde_json::to_string(&payload).unwrap_or_else(|_| \"{}\".into());\n+ Event::default().event(\"complete\").data(data)\n+}\n+\n+/// Stream `fetching` → `complete` / `error` for [`crate::ui_action::HtmlUiAction::FetchEntity`].\n+pub fn fetch_entity_stream(\n+ state: AppState,\n+ id: ItemId,\n+) -> Sse>> {\n+ tracing::debug!(item = %id, \"fetch entity stream opened\");\n+\n+ let stream = stream! {\n+ if id.is_root() {\n+ yield Ok(Event::default().event(\"error\").data(\"{\\\"message\\\":\\\"nothing to fetch for the root\\\"}\"));\n+ return;\n+ }\n+\n+ if !crate::reddit::is_fetchable(&id) {\n+ tracing::debug!(item = %id, \"fetch stream: not fetchable\");\n+ yield Ok(Event::default().event(\"error\").data(\"{\\\"message\\\":\\\"this page cannot be fetched from Reddit\\\"}\"));\n+ return;\n+ }\n+\n+ let fetching_html = {\n+ let tree = state.tree.read().await;\n+ let empty = NodeState::default();\n+ let node = tree.get(&id).unwrap_or(&empty);\n+ html::entity_section(&id, node, true).into_string()\n+ };\n+ let fetching_payload = serde_json::json!({\n+ \"selector\": \"#entity-section\",\n+ \"html\": fetching_html,\n+ });\n+ yield Ok(Event::default().event(\"fetching\").data(fetching_payload.to_string()));\n+\n+ let (tx, rx) = oneshot::channel();\n+ state.reddit.request_fetch(id.clone(), true, Some(tx));\n+ tracing::debug!(item = %id, \"fetch stream: queued reddit job\");\n+\n+ let result = match rx.await {\n+ Ok(r) => r,\n+ Err(_) => {\n+ tracing::warn!(item = %id, \"fetch stream: worker dropped oneshot\");\n+ FetchJobResult::Failed(\"reddit worker stopped\".into())\n+ }\n+ };\n+\n+ tracing::debug!(item = %id, ?result, \"fetch stream: job finished\");\n+\n+ match result {\n+ FetchJobResult::Imported | FetchJobResult::NotFound => {\n+ let tree = state.tree.read().await;\n+ let empty = NodeState::default();\n+ let node = tree.get(&id).unwrap_or(&empty);\n+ yield Ok(morph_complete_event(html::entity_section(&id, node, false)));\n+ }\n+ FetchJobResult::SkippedCached | FetchJobResult::SkippedDuplicate => {\n+ let tree = state.tree.read().await;\n+ let empty = NodeState::default();\n+ let node = tree.get(&id).unwrap_or(&empty);\n+ yield Ok(morph_complete_event(html::entity_section(&id, node, false)));\n+ }\n+ FetchJobResult::RateLimited { reset_secs } => {\n+ yield Ok(Event::default().event(\"error\").data(\n+ serde_json::json!({\"message\": format!(\"Reddit rate limit — retry in {reset_secs}s\")}).to_string(),\n+ ));\n+ }\n+ FetchJobResult::Failed(msg) => {\n+ yield Ok(Event::default().event(\"error\").data(\n+ serde_json::json!({\"message\": msg}).to_string(),\n+ ));\n+ }\n+ }\n+ };\n+\n+ Sse::new(stream).keep_alive(KeepAlive::new().interval(Duration::from_secs(15)))\n+}\ndiff --git a/server/src/html/mod.rs b/server/src/html/mod.rs\nindex db5b4c7f06b0be64603981166835cde268234f67..9314a7556306ddab969b896dbf4126b542a46722 100644\n--- a/server/src/html/mod.rs\n+++ b/server/src/html/mod.rs\n@@ -7,10 +7,10 @@ use axum::{\n use maud::{html, Markup, DOCTYPE};\n \n use crate::{\n+ fetch::html::entity_section,\n form_template::template_json_compact,\n path_types::ItemId,\n ranking::{top_bottom, RankedItem},\n- reddit::is_fetchable,\n reducer::{GroupState, NodeState},\n state::AppState,\n ui_action::UI_RPC_FIELD,\n@@ -149,62 +149,6 @@ pub fn breadcrumb_path(item: &ItemId) -> Markup {\n }\n }\n \n-fn entity_panel(node: &NodeState) -> Markup {\n- html! {\n- @if let Some(data) = &node.data {\n- div id=\"entity-panel\" class=\"entity-card\" {\n- h2 { (data.title) }\n- @if let Some(author) = &data.author {\n- p class=\"muted small\" { \"by \" (author) }\n- }\n- @if let Some(body) = &data.body_html {\n- div class=\"entity-body\" { (maud::PreEscaped(body)) }\n- }\n- }\n- }\n- }\n-}\n-\n-/// Reddit/API import control — only shown on fetchable pages; never auto-fires.\n-pub fn fetch_entity_panel(item: &ItemId, has_data: bool, fetching: bool) -> Markup {\n- if !is_fetchable(item) {\n- return html! {};\n- }\n- let label = if fetching {\n- \"Fetching…\"\n- } else if has_data {\n- \"Fetch more\"\n- } else {\n- \"Fetch from Reddit\"\n- };\n- let rpc = template_json_compact(&serde_json::json!({\n- \"action\": \"fetch_entity\",\n- \"item\": item.as_str(),\n- }))\n- .expect(\"fetch_entity rpc template\");\n- html! {\n- form method=\"post\" action=\"/ui\" id=\"fetch-entity-form\" class=\"fetch-entity-form\" {\n- input type=\"hidden\" name=(UI_RPC_FIELD) value=(rpc);\n- @if fetching {\n- button type=\"submit\" class=\"btn-secondary\" disabled { (label) }\n- } @else {\n- button type=\"submit\" class=\"btn-secondary\" { (label) }\n- }\n- }\n- }\n-}\n-\n-/// Entity card + explicit fetch control (morphed as `#entity-section`).\n-pub fn entity_section(item: &ItemId, node: &NodeState, fetching: bool) -> Markup {\n- let has_data = node.data.is_some();\n- html! {\n- section id=\"entity-section\" class=\"demo-panel\" {\n- (entity_panel(node))\n- (fetch_entity_panel(item, has_data, fetching))\n- }\n- }\n-}\n-\n fn rank_list(label: &str, items: &[RankedItem], start_rank: usize) -> Markup {\n html! {\n @if !items.is_empty() {\ndiff --git a/server/src/lib.rs b/server/src/lib.rs\nindex 79b173f391a96ae5d0d96fd656e1e8d2dd070d09..0677363e0ed21d244bfa065f7ec728549ddd9e50 100644\n--- a/server/src/lib.rs\n+++ b/server/src/lib.rs\n@@ -1,6 +1,7 @@\n pub mod api;\n pub mod event_log;\n pub mod events;\n+pub mod fetch;\n pub mod form_template;\n pub mod html;\n pub mod parser;\ndiff --git a/server/src/reddit.rs b/server/src/reddit.rs\nindex ff0f01e57b18af878eb5be3efc47204a7673589d..0e6ce32720951a58456852b67fb76a85dd9f4aad 100644\n--- a/server/src/reddit.rs\n+++ b/server/src/reddit.rs\n@@ -7,12 +7,12 @@ use std::time::{Duration, Instant};\n use reqwest::{header, Client, StatusCode};\n use serde::Deserialize;\n use serde_json::Value;\n-use tokio::sync::{mpsc, RwLock};\n+use tokio::sync::{mpsc, oneshot, RwLock};\n \n use crate::{\n event_log::EventLog,\n events::Event,\n- html::now_ms,\n+ fetch::now_ms,\n path_types::ItemId,\n reducer::GlobalTree,\n };\n@@ -22,10 +22,21 @@ pub fn ensure_partial_tree(tree: &mut GlobalTree, id: &ItemId) {\n tree.ensure_path(id);\n }\n \n+#[derive(Debug, Clone, PartialEq, Eq)]\n+pub enum FetchJobResult {\n+ Imported,\n+ NotFound,\n+ SkippedDuplicate,\n+ SkippedCached,\n+ RateLimited { reset_secs: u64 },\n+ Failed(String),\n+}\n+\n pub struct RedditCommand {\n pub id: ItemId,\n /// User-initiated fetch bypasses the in-memory \"recently fetched\" cache.\n pub force: bool,\n+ pub done: Option>,\n }\n \n #[derive(Clone)]\n@@ -72,14 +83,29 @@ impl RedditBroker {\n .build()\n .expect(\"reqwest client\");\n \n+ tracing::debug!(\n+ api_base = %config.api_base,\n+ oauth_base = %config.oauth_base,\n+ oauth = config.creds.is_some(),\n+ \"reddit worker started\"\n+ );\n+\n tokio::spawn(reddit_worker(rx, tree, event_log, client, config));\n \n Self { tx }\n }\n \n /// Queue a fetch; drops when the channel is full (backpressure).\n- pub fn request_fetch(&self, id: ItemId, force: bool) {\n- let _ = self.tx.try_send(RedditCommand { id, force });\n+ pub fn request_fetch(\n+ &self,\n+ id: ItemId,\n+ force: bool,\n+ done: Option>,\n+ ) {\n+ match self.tx.try_send(RedditCommand { id: id.clone(), force, done }) {\n+ Ok(()) => tracing::debug!(item = %id, force, \"reddit fetch queued\"),\n+ Err(_) => tracing::warn!(item = %id, \"reddit fetch queue full, dropped\"),\n+ }\n }\n }\n \n@@ -95,8 +121,6 @@ impl RedditApiConfig {\n }\n \n impl RedditCredentials {\n- /// Reddit's OAuth docs call these \"client id\" and \"client secret\"; the app\n- /// registration UI often labels them \"app id\" / \"app secret\" — same values.\n fn from_env() -> Option {\n let client_id = std::env::var(\"REDDIT_CLIENT_ID\")\n .or_else(|_| std::env::var(\"REDDIT_APP_ID\"))\n@@ -128,12 +152,10 @@ pub fn default_user_agent() -> String {\n })\n }\n \n-/// True when this node can be loaded from the Reddit JSON API.\n pub fn is_fetchable(id: &ItemId) -> bool {\n !map_item_to_reddit_api(id, \"https://example.com\").is_empty()\n }\n \n-/// Derive UI-facing fields from a stored payload (Reddit-specific when under reddit.com).\n pub fn entity_view_from_payload(id: &ItemId, payload: &Value) -> Option {\n if id.as_str().starts_with(\"reddit.com\") {\n return parse_reddit_view(id, payload);\n@@ -141,12 +163,17 @@ pub fn entity_view_from_payload(id: &ItemId, payload: &Value) -> Option>, result: FetchJobResult) {\n+ if let Some(tx) = done {\n+ let _ = tx.send(result);\n+ }\n+}\n+\n async fn reddit_worker(\n mut rx: mpsc::Receiver,\n tree: Arc>,\n@@ -168,15 +195,25 @@ async fn reddit_worker(\n recently_fetched.retain(|_, t| now.duration_since(*t) < cache_ttl);\n \n if in_flight.contains(&cmd.id) {\n+ tracing::debug!(item = %cmd.id, \"reddit fetch skipped: already in flight\");\n+ notify(cmd.done, FetchJobResult::SkippedDuplicate);\n continue;\n }\n if !cmd.force && recently_fetched.contains_key(&cmd.id) {\n+ tracing::debug!(item = %cmd.id, \"reddit fetch skipped: recently fetched cache\");\n+ notify(cmd.done, FetchJobResult::SkippedCached);\n continue;\n }\n \n in_flight.insert(cmd.id.clone());\n let fetch_id = cmd.id.clone();\n+ let done = cmd.done;\n \n+ tracing::debug!(\n+ item = %fetch_id,\n+ delay_ms = current_delay.as_millis(),\n+ \"reddit fetch starting after delay\"\n+ );\n tokio::time::sleep(current_delay).await;\n \n if let Some(c) = &creds {\n@@ -184,8 +221,18 @@ async fn reddit_worker(\n }\n \n let token = oauth.as_ref().map(|t| t.access_token.as_str());\n+ if token.is_some() {\n+ tracing::debug!(item = %fetch_id, \"reddit fetch using OAuth bearer\");\n+ }\n+\n+ let fetch_base = if token.is_some() {\n+ &oauth_base\n+ } else {\n+ &api_base\n+ };\n+ let outcome = do_fetch(&client, fetch_base, &fetch_id, token).await;\n \n- match do_fetch(&client, &api_base, &fetch_id, token).await {\n+ match outcome {\n Ok(FetchOutcome::Payload(payload)) => {\n let ts = now_ms();\n let event = Event::EntityImported {\n@@ -193,30 +240,49 @@ async fn reddit_worker(\n ts,\n payload: payload.clone(),\n };\n- if let Err(e) = event_log.append(&event).await {\n- tracing::warn!(\"event log append failed for {}: {}\", fetch_id, e);\n- } else {\n- apply_entity_import(&mut *tree.write().await, &fetch_id, payload);\n- recently_fetched.insert(fetch_id.clone(), Instant::now());\n- current_delay = Duration::from_millis(600);\n+ tracing::debug!(\n+ item = %fetch_id,\n+ ts,\n+ payload_keys = ?payload.as_object().map(|o| o.len()),\n+ \"reddit fetch got JSON payload, appending event\"\n+ );\n+ match event_log.append(&event).await {\n+ Err(e) => {\n+ tracing::warn!(\n+ item = %fetch_id,\n+ err = %e,\n+ \"reddit event log append failed\"\n+ );\n+ notify(done, FetchJobResult::Failed(e.to_string()));\n+ }\n+ Ok(()) => {\n+ apply_entity_import(&mut *tree.write().await, &fetch_id, payload);\n+ recently_fetched.insert(fetch_id.clone(), Instant::now());\n+ current_delay = Duration::from_millis(600);\n+ tracing::info!(item = %fetch_id, \"reddit entity imported\");\n+ notify(done, FetchJobResult::Imported);\n+ }\n }\n }\n Ok(FetchOutcome::NotFound) => {\n+ tracing::debug!(item = %fetch_id, \"reddit fetch: not found (no event written)\");\n recently_fetched.insert(fetch_id.clone(), Instant::now());\n+ notify(done, FetchJobResult::NotFound);\n }\n Ok(FetchOutcome::RateLimited { reset_secs }) => {\n- let wait = Duration::from_secs(reset_secs.max(1));\n tracing::warn!(\n- \"Reddit rate limit for {}; sleeping {}s\",\n- fetch_id,\n- wait.as_secs()\n+ item = %fetch_id,\n+ reset_secs,\n+ \"reddit rate limited\"\n );\n- tokio::time::sleep(wait).await;\n+ tokio::time::sleep(Duration::from_secs(reset_secs.max(1))).await;\n current_delay = (current_delay * 2).min(Duration::from_secs(60));\n+ notify(done, FetchJobResult::RateLimited { reset_secs });\n }\n Err(e) => {\n- tracing::warn!(\"Reddit fetch failed for {}: {}\", fetch_id, e);\n+ tracing::warn!(item = %fetch_id, err = %e, \"reddit fetch failed\");\n current_delay = (current_delay * 2).min(Duration::from_secs(60));\n+ notify(done, FetchJobResult::Failed(e));\n }\n }\n \n@@ -238,6 +304,7 @@ async fn ensure_oauth_token(\n ) -> Option {\n if let Some(t) = existing {\n if Instant::now() < t.expires_at - Duration::from_secs(60) {\n+ tracing::debug!(\"reddit OAuth token still valid\");\n return Some(t);\n }\n }\n@@ -246,6 +313,7 @@ async fn ensure_oauth_token(\n \"{}/api/v1/access_token\",\n oauth_base.trim_end_matches('/')\n );\n+ tracing::debug!(%url, \"reddit OAuth token request\");\n \n let resp = client\n .post(&url)\n@@ -257,13 +325,13 @@ async fn ensure_oauth_token(\n let resp = match resp {\n Ok(r) => r,\n Err(e) => {\n- tracing::warn!(\"Reddit OAuth token request failed: {e}\");\n+ tracing::warn!(\"reddit OAuth token request failed: {e}\");\n return None;\n }\n };\n \n if !resp.status().is_success() {\n- tracing::warn!(\"Reddit OAuth token HTTP {}\", resp.status());\n+ tracing::warn!(\"reddit OAuth token HTTP {}\", resp.status());\n return None;\n }\n \n@@ -276,11 +344,12 @@ async fn ensure_oauth_token(\n let body: TokenResponse = match resp.json().await {\n Ok(b) => b,\n Err(e) => {\n- tracing::warn!(\"Reddit OAuth token parse failed: {e}\");\n+ tracing::warn!(\"reddit OAuth token parse failed: {e}\");\n return None;\n }\n };\n \n+ tracing::debug!(expires_in = body.expires_in, \"reddit OAuth token acquired\");\n Some(OAuthToken {\n access_token: body.access_token,\n expires_at: Instant::now() + Duration::from_secs(body.expires_in),\n@@ -295,35 +364,72 @@ async fn do_fetch(\n ) -> Result {\n let url = map_item_to_reddit_api(id, api_base);\n if url.is_empty() {\n+ tracing::debug!(item = %id, \"reddit do_fetch: no API URL for item\");\n return Ok(FetchOutcome::NotFound);\n }\n \n+ tracing::debug!(item = %id, %url, bearer = bearer.is_some(), \"reddit HTTP GET\");\n+\n let mut req = client.get(&url);\n if let Some(token) = bearer {\n req = req.bearer_auth(token);\n }\n \n- let resp = req.send().await.map_err(|e| e.to_string())?;\n+ let resp = req.send().await.map_err(|e| {\n+ tracing::debug!(item = %id, %url, err = %e, \"reddit HTTP transport error\");\n+ e.to_string()\n+ })?;\n+\n+ let status = resp.status();\n+ tracing::debug!(\n+ item = %id,\n+ %url,\n+ %status,\n+ remaining = ?rate_limit_remaining(&resp),\n+ reset = ?rate_limit_reset_secs(&resp),\n+ \"reddit HTTP response\"\n+ );\n \n- if resp.status() == StatusCode::TOO_MANY_REQUESTS {\n+ if status == StatusCode::TOO_MANY_REQUESTS {\n let reset = rate_limit_reset_secs(&resp);\n return Ok(FetchOutcome::RateLimited { reset_secs: reset });\n }\n \n- if resp.status() == StatusCode::SERVICE_UNAVAILABLE {\n- return Err(\"Reddit unavailable (503)\".to_string());\n+ if status == StatusCode::SERVICE_UNAVAILABLE {\n+ return Err(\"Reddit unavailable (503)\".into());\n }\n \n- if !resp.status().is_success() {\n+ if !status.is_success() {\n+ let body = resp.text().await.unwrap_or_default();\n+ tracing::debug!(\n+ item = %id,\n+ %status,\n+ body_len = body.len(),\n+ body_prefix = %body.chars().take(240).collect::(),\n+ \"reddit non-success body\"\n+ );\n return Ok(FetchOutcome::NotFound);\n }\n \n if rate_limit_remaining(&resp) == Some(0) {\n let reset = rate_limit_reset_secs(&resp);\n+ tracing::debug!(item = %id, reset_secs = reset, \"reddit headers: rate limit exhausted\");\n return Ok(FetchOutcome::RateLimited { reset_secs: reset });\n }\n \n- let payload: Value = resp.json().await.map_err(|e| e.to_string())?;\n+ let text = resp.text().await.map_err(|e| e.to_string())?;\n+ tracing::debug!(item = %id, bytes = text.len(), \"reddit response body received\");\n+\n+ let payload: Value = serde_json::from_str(&text).map_err(|e| {\n+ tracing::debug!(\n+ item = %id,\n+ err = %e,\n+ body_prefix = %text.chars().take(240).collect::(),\n+ \"reddit JSON parse failed\"\n+ );\n+ format!(\"invalid JSON: {e}\")\n+ })?;\n+\n Ok(FetchOutcome::Payload(payload))\n }\n \n@@ -344,7 +450,6 @@ fn rate_limit_reset_secs(resp: &reqwest::Response) -> u64 {\n .unwrap_or(5)\n }\n \n-/// Map canonical item id to a Reddit JSON API URL under `api_base`.\n pub fn map_item_to_reddit_api(id: &ItemId, api_base: &str) -> String {\n let path = id.as_str();\n if !path.starts_with(\"reddit.com/\") && path != \"reddit.com\" {\n@@ -445,26 +550,6 @@ mod tests {\n map_item_to_reddit_api(&id, \"https://www.reddit.com\"),\n \"https://www.reddit.com/r/rust/about.json?raw_json=1\"\n );\n- assert_eq!(\n- map_item_to_reddit_api(&id, \"http://127.0.0.1:9999\"),\n- \"http://127.0.0.1:9999/r/rust/about.json?raw_json=1\"\n- );\n- }\n-\n- #[test]\n- fn map_post_url() {\n- let id = ItemId::parse(\"reddit.com/r/amitheasshole/comments/1trnvdl\").unwrap();\n- assert_eq!(\n- map_item_to_reddit_api(&id, \"https://www.reddit.com\"),\n- \"https://www.reddit.com/r/amitheasshole/comments/1trnvdl.json?raw_json=1\"\n- );\n- }\n-\n- #[test]\n- fn is_fetchable_reddit_sub() {\n- let id = ItemId::parse(\"reddit.com/r/rust\").unwrap();\n- assert!(is_fetchable(&id));\n- assert!(!is_fetchable(&ItemId::opaque(\"example.com/x\")));\n }\n \n #[test]\n@@ -477,19 +562,5 @@ mod tests {\n )\n .unwrap();\n assert_eq!(entity.title, \"The Rust Programming Language\");\n- assert!(entity.body_html.as_ref().is_some_and(|b| b.contains(\"Rust\")));\n- }\n-\n- #[test]\n- fn parse_post_fixture() {\n- let json = r#\"[{\"kind\":\"Listing\",\"data\":{\"children\":[{\"kind\":\"t3\",\"data\":{\"title\":\"AITA\",\"author\":\"op\",\"selftext_html\":\"<p>hi</p>\",\"thumbnail\":\"https://b.thumbs.redditmedia.com/x.jpg\"}}]}}]\"#;\n- let v: Value = serde_json::from_str(json).unwrap();\n- let entity = entity_view_from_payload(\n- &ItemId::parse(\"reddit.com/r/x/comments/abc\").unwrap(),\n- &v,\n- )\n- .unwrap();\n- assert_eq!(entity.title, \"AITA\");\n- assert_eq!(entity.author.as_deref(), Some(\"op\"));\n }\n }\ndiff --git a/server/src/state.rs b/server/src/state.rs\nindex d71d1079a8f486cdab15795384aef0b81b32544d..e7ff9f663e45b5e168d6a3869948e3bd890966d4 100644\n--- a/server/src/state.rs\n+++ b/server/src/state.rs\n@@ -143,9 +143,9 @@ impl AppState {\n Ok(())\n }\n \n- /// User-initiated Reddit/API import (via \"Fetch more\" — never on paste or navigate).\n- pub fn queue_entity_fetch(&self, id: ItemId) {\n- self.reddit.request_fetch(id, true);\n+ /// User-initiated Reddit/API import (SSE / fetch module only).\n+ pub fn queue_entity_fetch(&self, id: ItemId, done: Option>) {\n+ self.reddit.request_fetch(id, true, done);\n }\n \n pub async fn record_vote(\ndiff --git a/server/src/ui_action.rs b/server/src/ui_action.rs\nindex 3d6a49a2a3fb950752827efe1f5a049308f09baf..ef9ac873fc0442e0b960f144309c8e91124d7884 100644\n--- a/server/src/ui_action.rs\n+++ b/server/src/ui_action.rs\n@@ -26,7 +26,7 @@ pub enum HtmlUiAction {\n ParseQuery {\n query: String,\n },\n- /// Fetch upstream entity data for the current page (explicit user action only).\n+ /// Import entity data; `POST /ui` responds with `text/event-stream` (not JS).\n FetchEntity {\n item: String,\n },\ndiff --git a/server/static/sorter_ui.js b/server/static/sorter_ui.js\nindex d9b016f197547f37ca4b2bcdd7ee6b673d0fb3f0..fe7bfc657c0fccae5c596d005b0e807abefec677 100644\n--- a/server/static/sorter_ui.js\n+++ b/server/static/sorter_ui.js\n@@ -1,5 +1,5 @@\n /**\n- * sorter2 web UI: fetch/eval for POST /ui. No product logic here.\n+ * sorter2 web UI: POST /ui returns JS (morph) or SSE (entity fetch).\n */\n (function () {\n function evalJs(js) {\n@@ -8,15 +8,98 @@\n }\n }\n \n+ function morphSelector(selector, html) {\n+ var el = document.querySelector(selector);\n+ if (el && typeof Idiomorph !== 'undefined') {\n+ Idiomorph.morph(el, html);\n+ }\n+ }\n+\n+ function handleSseEvent(eventType, data, form) {\n+ if (eventType === 'fetching' || eventType === 'complete') {\n+ try {\n+ var msg = JSON.parse(data);\n+ morphSelector(msg.selector || '#entity-section', msg.html);\n+ } catch (err) {\n+ console.warn('fetch morph parse', err);\n+ }\n+ }\n+ if (eventType === 'complete' || eventType === 'error') {\n+ var btn = form && form.querySelector('button[type=\"submit\"]');\n+ if (btn) btn.disabled = false;\n+ }\n+ if (eventType === 'error') {\n+ try {\n+ var err = JSON.parse(data);\n+ console.warn('fetch error:', err.message || data);\n+ } catch (_e) {\n+ console.warn('fetch error:', data);\n+ }\n+ }\n+ }\n+\n+ function consumeSseStream(response, form) {\n+ var reader = response.body.getReader();\n+ var decoder = new TextDecoder();\n+ var buffer = '';\n+ var eventType = '';\n+ var dataLines = [];\n+\n+ function dispatch() {\n+ if (!eventType && dataLines.length === 0) return;\n+ handleSseEvent(eventType || 'message', dataLines.join('\\n'), form);\n+ eventType = '';\n+ dataLines = [];\n+ }\n+\n+ function pump() {\n+ return reader.read().then(function (chunk) {\n+ if (chunk.done) {\n+ dispatch();\n+ return;\n+ }\n+ buffer += decoder.decode(chunk.value, { stream: true });\n+ var parts = buffer.split('\\n');\n+ buffer = parts.pop() || '';\n+ for (var i = 0; i < parts.length; i++) {\n+ var line = parts[i].replace(/\\r$/, '');\n+ if (line === '') {\n+ dispatch();\n+ } else if (line.indexOf('event:') === 0) {\n+ eventType = line.slice(6).trim();\n+ } else if (line.indexOf('data:') === 0) {\n+ dataLines.push(line.slice(5).trim());\n+ }\n+ }\n+ return pump();\n+ });\n+ }\n+\n+ return pump();\n+ }\n+\n function postUiForm(form) {\n+ var btn = form.querySelector('button[type=\"submit\"]');\n+ if (form.id === 'fetch-entity-form' && btn) {\n+ btn.disabled = true;\n+ }\n return fetch(form.action, {\n method: 'POST',\n body: new URLSearchParams(new FormData(form)),\n headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n credentials: 'same-origin',\n }).then(function (resp) {\n- return resp.text();\n- }).then(evalJs);\n+ var ct = resp.headers.get('content-type') || '';\n+ if (ct.indexOf('text/event-stream') !== -1) {\n+ return consumeSseStream(resp, form);\n+ }\n+ return resp.text().then(evalJs);\n+ }).catch(function (err) {\n+ if (form.id === 'fetch-entity-form' && btn) {\n+ btn.disabled = false;\n+ }\n+ console.warn('POST /ui failed', err);\n+ });\n }\n \n function initSorterUi() {\ndiff --git a/test/reddit_import.clj b/test/reddit_import.clj\nindex 54edaedeef08718c8f184d6aa468d8e6415068c7..17f2ee39e1498b0b6bb6a9d8a7e35608b0bd1b32 100644\n--- a/test/reddit_import.clj\n+++ b/test/reddit_import.clj\n@@ -44,10 +44,12 @@\n (do (Thread/sleep 200) (recur))\n false))))))\n \n-(defn- curl-post-ui [base rpc-json]\n+(defn- curl-fetch-ui-sse [base item]\n (process/shell {:out :string :err :string}\n- \"curl\" \"-sf\" \"-X\" \"POST\" (str base \"/ui\")\n- \"--data-urlencode\" (str \"__rpc__=\" rpc-json)))\n+ \"curl\" \"-sfN\" \"--max-time\" \"20\"\n+ \"-X\" \"POST\" (str base \"/ui\")\n+ \"--data-urlencode\"\n+ (str \"__rpc__={\\\"action\\\":\\\"fetch_entity\\\",\\\"item\\\":\\\"\" item \"\\\"}\")))\n \n (defn- wait-event-log [path ms]\n (let [deadline (+ (System/currentTimeMillis) ms)]\n@@ -98,11 +100,12 @@\n \"curl\" \"-sf\" browse-url))]\n (is (str/includes? before \"Fetch from Reddit\"))\n (is (not (str/includes? before \"The Rust Programming Language\")))\n- (let [rpc \"{\\\"action\\\":\\\"fetch_entity\\\",\\\"item\\\":\\\"reddit.com/r/rust\\\"}\"\n- post (curl-post-ui app-base rpc)\n- log-path (str data-dir \"/events.jsonl\")]\n- (is (zero? (:exit post)) \"fetch_entity POST succeeds\")\n- (is (wait-event-log log-path 10000) \"event log written\")\n+ (let [log-path (str data-dir \"/events.jsonl\")\n+ sse (curl-fetch-ui-sse app-base \"reddit.com/r/rust\")]\n+ (is (zero? (:exit sse)) \"POST /ui fetch_entity SSE succeeds\")\n+ (is (str/includes? (:out sse) \"event: complete\"))\n+ (is (str/includes? (:out sse) \"The Rust Programming Language\"))\n+ (is (wait-event-log log-path 2000) \"event log written\")\n (let [after (:out (process/shell {:out :string :err :string}\n \"curl\" \"-sf\" browse-url))\n log (slurp (io/file log-path))]\n","role":"user"}],"model":"openai/gpt-chat-latest"}