Commit B replaces an ad-hoc rule-combinator engine with a more principled semantic graph/DFA architecture for URL canonicalization, a substantive design change with lasting structural impact (even though the diff removes tests, it references new modules graph.rs/parse.rs/registry_tests.rs implying continued coverage). Commit A is a smaller correctness/clarity fix (converting a silent fallback to an explicit panic with better invariants and doc clarifications, plus added test assertions), which is valuable but narrower in scope than B's architectural overhaul.
constitution · epochs · watch · epoch 3
comparison
c_6a02ffb06a41 (tommy-mor) vs c_f10e7b043e68 (tommy-mor)
download prompt · raw event · cmp_45f6897e4e8aa5
council reasoning
B replaces ad-hoc URL combinators (engine.rs + per-host normalize paths) with a graph/DFA-based canonicalization API wired through registry, which is core identity infrastructure lasting across the product. A is a correct but narrow fix: rank-history thread_post_index becomes strictly 0-based with expect() and tests, removing optional 1-based/unwrap_or(0) mismatch—valuable polish, not system-level design.
Side A fixes the thread post index semantics by making rank history consistently use the 0-based URL index, replacing silent `unwrap_or(0)` fallbacks with `expect(...)` to catch broken invariants, updating documentation, removing conditional rendering, and adding integration tests that verify the behavior. Side B is primarily a large refactor of the URL canonicalization subsystem (moving logic into new graph/parse modules and deleting the old engine) but the shown patch mostly rewires interfaces and removes in-file tests without exposing the new implementation, making its lasting functional value less evident from the diff.
sides
A — c_6a02ffb06a41 (tommy-mor)
message
[3bc88847] removed optional
diff preview
diff --git a/server/src/api/rpc.rs b/server/src/api/rpc.rs
index ba93ca87182d49f57dffc8220f604ffa150f9a6c..5049a26b096bd8435d6eb9e75ccb751b6f489061 100644
--- a/server/src/api/rpc.rs
+++ b/server/src/api/rpc.rs
@@ -1337,8 +1337,7 @@ pub async fn handle_rpc_batch(
.ingests_by_scope_thread
.get(&(scope.clone(), e.thread.clone()))
.and_then(|q| q.iter().rev().position(|id| id == &e.post_id))
- .map(|i| i + 1)
- .unwrap_or(0);
+ .expect("rank history post_id must be in ingests_by_scope_thread for (scope, thread)");
RankHistoryRow {
ts: e.ts,
scope_rank: e.scope_rank,
diff --git a/server/src/html/garden.rs b/server/src/html/garden.rs
index 319feb15a6b68d2b5df98b4289fedbc9bdd048d3..23245d6fdfc9019b199ab8417150faf5f3297067 100644
--- a/server/src/html/garden.rs
+++ b/server/src/html/garden.rs
@@ -450,6 +450,7 @@ struct RankHistoryEntryView {
scope_total: usize,
scope_rank_delta: i32,
thread: String,
+ /// 0-based index as [`crate::html::forum::ingest::thread_post_index_in_scope`] / `/t/tag/N`.
thread_post_index: usize,
caused_by: Vec<crate::reducer::VoteData>,
}
@@ -573,11 +574,11 @@ fn build_rank_history(
})
.unwrap_or_default();
- let thread_post_index = reduced.ingests_by_scope_thread
+ let thread_post_index = reduced
+ .ingests_by_scope_thread
.get(&(scope.clone(), e.thread.clone()))
.and_then(|q| q.iter().rev().position(|id| id == &e.post_id))
- .map(|i| i + 1)
- .unwrap_or(0);
+ .expect("rank history post_id must be in ingests_by_scope_thread for (scope, thread)");
RankHistoryEntryView {
ts: e.ts,
@@ -704,11 +705,9 @@ async fn render_scope_view(
span class="muted" { (ago) (label) }
" · "
a href=(thread_href(&e.thread)) { "#" (e.thread) }
- @if e.thread_post_index > 0 {
- " "
- a href=(format!("{}/{}", thread_href(&e.thread), e.thread_post_index)) {
- span class="muted" { "post #" (e.thread_post_index) }
- }
+ " "
+ a href=(format!("{}/{}", thread_href(&e.thread), e.thread_post_index)) {
+ span class="muted" { "post #" (e.thread_post_index) }
}
}
@if e.caused_by.is_empty() {
diff --git a/server/tests/integration.rs b/server/tests/integration.rs
index d4c5bfe9c6f1c71dc61878bd8c5e729b1b7c69ef..9766adb43ad315a64f5df17b79f66718ce149509 100644
--- a/server/tests/integration.rs
+++ b/server/tests/integration.rs
@@ -1322,6 +1322,11 @@ async fn test_rank_history() {
assert_eq!(entry["scope_rank_delta"], 0, "delta is 0 on first appearance");
let caused_by = entry["caused_by"].as_array().unwrap();
assert_eq!(caused_by.len(), 2, "both votes in the ingest touched rust");
+ assert_eq!(
+ entry["thread_post_index"],
+ 0,
+ "rank history links use same 0-based index as /t/hist-test/0"
+ );
ingest(
"00000000-0000-0000-0000-000000000002:rig:test/model",
@@ -1349,6 +1354,16 @@ async fn test_rank_history() {
assert_eq!(caused_by2.len(), 1);
assert!(caused_by2[0]["a"].as_str().unwrap().ends_with("python") ||
caused_by2[0]["b"].as_str().unwrap().ends_with("python"));
+ assert_eq!(
+ hist2[0]["thread_post_index"],
+ 0,
+ "first hist-test post is chronological index 0"
+ );
+ assert_eq!(
+ hist2[1]["thread_post_index"],
+ 1,
+ "second ingest is chronological index 1"
+ );
let hist_rust2 = rpc_batch(
&client,
diff --git a/types/src/lib.rs b/types/src/lib.rs
index edbb923e4d7d41ad82dfc254c3bd697562383527..49abba8ea9f786ef68e3157d2a5d309e15e09ba0 100644
--- a/types/src/lib.rs
+++ b/types/src/lib.rs
@@ -253,7 +253,7 @@ pub struct FeedPost {
/// Primary thread tag (without #), if the ingest declared one.
#[serde(skip_serializing_if = "Option::is_none")]
pub thread: Option<String>,
- /// 1-indexed chronological position of this post within the thread.
+ /// 1-based display ordinal for this post within the thread (feed only; URLs use 0-based paths).
#[serde(skip_serializing_if = "Option::is_none")]
pub thread_post_index: Option<usize>,
/// Full raw body of the ingest document.
@@ -628,7 +628,7 @@ pub struct RankHistoryRow {
pub score: f64,
/// Thread tag of the ingest that triggered this rank change.
pub thread: String,
- /// 1-indexed chronological position of this post within the thread.
+ /// 0-indexed chronological position of this post within the thread (same as `/t/tag/N` routes).
pub thread_post_index: usize,
/// Votes from this ingest that directly touched this item. Empty when change was transitive.
pub caused_by: Vec<VoteRow>,
B — c_f10e7b043e68 (tommy-mor)
message
[7bb7145d] url stuff
diff preview
diff --git a/AGENTS.md b/AGENTS.md
index e60b9ba6012593361ef10e8fdd9439cd9932e09b..babb889d6fbfb1fa7176c9e6b7544ae17b61dd2e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -58,4 +58,4 @@ Use **tmux** for `cargo run --package sorter2-server` (dev server). Rebuild afte
- First `cargo test` / `cargo build --release` is slow; Clojure smoke test always does a release build.
- `legacy/` and `ideas/` are not part of the workspace build.
-- **ItemId** for web URLs is a canonical full URL (`https://reddit.com/r/rust`). Rules live in [`server/src/url_rules/`](server/src/url_rules/) (composable Rust, not a config DSL). After changing canonicalization rules, rebuild the projection: `cargo run --package sorter2-server -- replay-index`.
+- **ItemId** for web URLs is a canonical full URL (`https://reddit.com/r/rust`). Rules live in [`server/src/url_rules/graph.rs`](server/src/url_rules/graph.rs): a semantic graph (DFA on host + path, query params in `Context`) with a generic internet fallback for unknown sites. After changing rules, rebuild the projection: `cargo run --package sorter2-server -- replay-index`.
diff --git a/server/src/url_rules/engine.rs b/server/src/url_rules/engine.rs
deleted file mode 100644
index e29b6b48c08deb7bffe031b1e542b1e25a7bef15..0000000000000000000000000000000000000000
--- a/server/src/url_rules/engine.rs
+++ /dev/null
@@ -1,187 +0,0 @@
-//! Composable URL normalization primitives.
-
-use std::collections::HashMap;
-
-use url::Url;
-
-/// Mutable URL view used by rule combinators before serializing to a canonical string.
-#[derive(Debug, Clone)]
-pub struct ParsedUrl {
- pub scheme: String,
- pub host: String,
- pub path_segments: Vec<String>,
- pub query: HashMap<String, String>,
- pub fragment: Option<String>,
-}
-
-impl ParsedUrl {
- pub fn parse(raw: &str) -> Option<Self> {
- let trimmed = raw.trim();
- if trimmed.is_empty() {
- return None;
- }
-
- let with_scheme = if trimmed.contains("://") {
- trimmed.to_string()
- } else if trimmed.starts_with("r/") || trimmed.starts_with("/r/") {
- let rest = trimmed.trim_start_matches('/').trim_start_matches("r/");
- format!("https://reddit.com/r/{rest}")
- } else if trimmed.contains('.') && !trimmed.starts_with('/') {
- format!("https://{trimmed}")
- } else {
- trimmed.to_string()
- };
-
- let url = Url::parse(&with_scheme).ok()?;
- let host = url.host_str()?.to_string();
- let path_segments: Vec<String> = url
- .path_segments()
- .map(|segs| segs.filter(|s| !s.is_empty()).map(str::to_string).collect())
- .unwrap_or_default();
-
- let mut query = HashMap::new();
- for (k, v) in url.query_pairs() {
- query.insert(k.into_owned(), v.into_owned());
- }
-
- Some(Self {
- scheme: url.scheme().to_string(),
- path_segments,
- query,
- fragment: url.fragment().map(str::to_string),
- host,
- })
- }
-
- pub fn with_path_segments(&self, segments: &[String]) -> Self {
- let mut u = self.clone();
- u.path_segments = segments.to_vec();
- u
- }
-
- pub fn to_url(&self) -> Option<Url> {
- let mut url = if self.path_segments.is_empty() {
- Url::parse(&format!("{}://{}", self.scheme, self.host)).ok()?
- } else {
- let path = format!("/{}", self.path_segments.join("/"));
- Url::parse(&format!("{}://{}{}", self.scheme, self.host, path)).ok()?
- };
- if !self.query.is_empty() {
- let mut pairs: Vec<_> = self.query.iter().collect();
- pairs.sort_by(|a, b| a.0.cmp(b.0));
- url.query_pairs_mut().clear();
- for (k, v) in pairs {
- url.query_pairs_mut().append_pair(k, v);
- }
- }
- if let Some(ref frag) = self.fragment {
- url.set_fragment(Some(frag));
- }
- Some(url)
- }
-
- pub fn canonical_string(&self) -> Option<String> {
- let url = self.to_url()?;
- let mut s = url.to_string();
- if self.path_segments.is_empty() {
- s = s.trim_end_matches('/').to_string();
- }
- Some(s)
- }
-}
-
-pub fn force_https(u: &mut ParsedUrl) {
- if u.scheme == "http" {
- u.scheme = "https".to_string();
- }
-}
-
-pub fn drop_fragment(u: &mut ParsedUrl) {
- u.fragment = None;
-}
-
-pub fn strip_www(u: &mut ParsedUrl) {
- if u.host.starts_with("www.") {
- u.host = u.host[4..].to_string();
- }
-}
-
-pub fn lowercase_host(u: &mut ParsedUrl) {
- u.host = u.host.to_ascii_lowercase();
-}
-
-pub fn lowercase_path(u: &mut ParsedUrl) {
- for seg in &mut u.path_segments {
- *seg = seg.to_ascii_lowercase();
- }
-}
-
-pub fn clear_query(u: &mut ParsedUrl) {
- u.query.clear();
-}
-
-pub fn keep_only_query(u: &mut ParsedUrl, keys: &[&str]) {
- u.query
- .retain(|k, _| keys.iter().any(|want| want == &k.as_str()));
-}
-
-pub fn strip_tracking_params(u: &mut ParsedUrl) {
- u.query.retain(|k, _| {
- let lower = k.to_ascii_lowercase();
- !(lower.starts_with("utm_")
- || matches!(
- lower.as_str(),
- "fbclid" | "gclid" | "ref" | "ref_src" | "ref_source" | "mc_cid" | "mc_eid"
- ))
- });
-}
-
-pub fn truncate_after_segment(u: &mut ParsedUrl, name: &str, keep: usize) {
- if let Some(i) = u.path_segments.iter().position(|s| s == name) {
- let end = (i + 1 + keep).min(u.path_segments.len());
- u.path_segments.truncate(end);
- }
-}
-
-pub fn drop_listing_suffix(u: &mut ParsedUrl, suffixes: &[&str]) {
- if u.path_segments.len() >= 3 && u.path_segments.first().map(String::as_str) == Some("r") {
- if let Some(last) = u.path_segments.last() {
- if suffixes.iter().any(|s| *s == last.as_str()) {
- u.path_segments.pop();
- }
- }
- }
-}
-
-pub fn normalize_reddit_host(u: &mut ParsedUrl) {
- if matches!(
- u.host.as_str(),
- "old.reddit.com" | "new.reddit.com" | "www.reddit.com"
- ) {
- u.host = "reddit.com".to_string();
- }
-}
-
-pub fn rewrite_youtu_be(u: &mut ParsedUrl) {
- if u.host == "youtu.be" && u.path_segments.len() == 1 {
- let id = u.path_segments[0].clone();
- u.host = "youtube.com".to_string();
- u.path_segments = vec!["watch".to_string()];
- u.query.insert("v".to_string(), id);
- }
-}
-
-pub fn rewrite_youtube_shorts(u: &mut ParsedUrl) {
- if u.host == "youtube.com" && u.path_segments.first().map(String::as_str) == Some("shorts") {
- if let Some(id) = u.path_segments.get(1).cloned() {
- u.path_segments = vec!["watch".to_string()];
- u.query.insert("v".to_string(), id);
- }
- }
-}
-
-pub fn normalize_youtube_host(u: &mut ParsedUrl) {
- if matches!(u.host.as_str(), "m.youtube.com" | "www.youtube.com") {
- u.host = "youtube.com".to_string();
- }
-}
diff --git a/server/src/url_rules/mod.rs b/server/src/url_rules/mod.rs
index 03d53bd3e82d704a01ba3fd8dd02b7d31422c0de..9e1445346ce77a49dd6a7e7713bf9c57aef353cc 100644
--- a/server/src/url_rules/mod.rs
+++ b/server/src/url_rules/mod.rs
@@ -1,8 +1,12 @@
-//! URL canonicalization and hierarchy rules for [`crate::path_types::ItemId`].
+//! URL canonicalization and hierarchy via a semantic graph (DFA + generic fallback).
-mod engine;
+mod graph;
+mod parse;
mod registry;
+#[cfg(test)]
+mod registry_tests;
+
pub use registry::{
canonicalize_raw, looks_like_url, navigable_breadcrumbs, parent_url, resolve_id, CanonicalResult,
};
diff --git a/server/src/url_rules/registry.rs b/server/src/url_rules/registry.rs
index 14514e9af8385fb2b9b2f35eb9ee14d453d4b97c..8e6c012ea1fc74b864307bdacdf5a0f5db5259fc 100644
--- a/server/src/url_rules/registry.rs
+++ b/server/src/url_rules/registry.rs
@@ -1,12 +1,7 @@
-//! Per-domain canonicalization and hierarchy rules.
+//! Public API: canonical identity and hierarchy via the URL graph.
-use std::collections::HashSet;
-
-use super::engine::{
- clear_query, drop_fragment, drop_listing_suffix, force_https, keep_only_query, lowercase_host,
- lowercase_path, normalize_reddit_host, normalize_youtube_host, rewrite_youtu_be,
- rewrite_youtube_shorts, strip_tracking_params, strip_www, truncate_after_segment, ParsedUrl,
-};
+use super::graph::graph;
+use super::parse::UrlParts;
/// Result of canonicalizing a raw URL string.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -16,71 +11,16 @@ pub struct CanonicalResult {
pub alias_of: Option<String>,
}
-fn apply_global(u: &mut ParsedUrl) {
- force_https(u);
- drop_fragment(u);
- strip_www(u);
- lowercase_host(u);
- strip_tracking_params(u);
-}
-
-fn normalize_reddit(u: &mut ParsedUrl) {
- normalize_reddit_host(u);
- lowercase_path(u);
- truncate_after_segment(u, "comments", 1);
- drop_listing_suffix(u, &["hot", "top", "new", "rising", "controversial"]);
- clear_query(u);
-}
-
-fn normalize_youtube(u: &mut ParsedUrl) {
- rewrite_youtu_be(u);
- normalize_youtube_host(u);
- rewrite_youtube_shorts(u);
- keep_only_query(u, &["v", "list"]);
-}
-
-fn normalize_default(_u: &mut ParsedUrl) {
- // Global rules only.
-}
-
-fn domain_key(host: &str) -> &'static str {
- if host == "reddit.com" || host.ends_with(".reddit.com") {
- "reddit.com"
- } else if host == "youtube.com" || host == "youtu.be" {
- "youtube.com"
- } else {
- "default"
- }
-}
-
-fn normalize_for_host(u: &mut ParsedUrl) {
- apply_global(u);
- match domain_key(&u.host) {
- "reddit.com" => normalize_reddit(u),
- "youtube.com" => normalize_youtube(u),
- _ => normalize_default(u),
- }
-}
-
-/// Structural path segments that must not become standalone tree nodes when more path follows.
-fn structural_trailing(host: &str) -> &'static [&'static str] {
- match domain_key(host) {
- "reddit.com" => &["comments"],
- _ => &[],
- }
-}
-
/// Canonicalize a raw URL. Returns `None` if the input is not URL-like.
pub fn canonicalize_raw(raw: &str) -> Option<CanonicalResult> {
let trimmed = raw.trim();
if trimmed.is_empty() {
return None;
}
- let mut u = ParsedUrl::parse(trimmed)?;
- let input_snapshot = u.canonical_string()?;
- normalize_for_host(&mut u);
- let canonical = u.canonical_string()?;
- let alias_of = if input_snapshot != canonical {
+ let parts = UrlParts::parse(trimmed)?;
+ let g = graph();
+ let canonical = g.resolve_canonical(&parts)?;
+ let alias_of = if trimmed != canonical {
Some(trimmed.to_string())
} else {
None
@@ -98,35 +38,14 @@ pub fn resolve_id(raw: &str) -> Option<String> {
/// Navigable ancestor URLs from domain root up to and including `canonical` (full URLs).
pub fn navigable_breadcrumbs(canonical: &str) -> Vec<String> {
- let Some(u) = ParsedUrl::parse(canonical) else {
- return vec![canonical.to_string()];
+ let parts = match UrlParts::parse(canonical) {
+ Some(p) => p,
+ None => return vec![canonical.to_string()],
};
- let structural: HashSet<&str> = structural_trailing(&u.host).iter().copied().collect();
- let n = u.path_segments.len();
- let mut out = Vec::new();
-
- // Domain root (no path segments).
- if let Some(base) = u.with_path_segments(&[]).canonical_string() {
- out.push(base);
- }
-
- for i in 0..n {
- let segs: Vec<String> = u.path_segments[..=i].to_vec();
- let is_last = i == n - 1;
- let seg = u.path_segments[i].as_str();
- if structural.contains(seg) && !is_last {
- continue;
- }
- if let Some(url) = u.with_path_segments(&segs).canonical_string() {
- if out.last() != Some(&url) {
-
… preview truncated; 3,144 characters omittedHardlinks — judgments / attempts / prompt
judgments
attempts
Prompt text is loaded only by the download route.