Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 58 additions & 4 deletions backend/crates/itcy/src/sources/publisher_url.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use crate::sources::draft_footer::ensure_primary_link_line;
use crate::sources::draft_url::{extract_in_post_url, set_single_in_post_url};
use crate::sources::html::extract_articleish_text;
use crate::sources::tweet_footer::extract_brief_cite;
use crate::sources::url_hygiene::{
is_allowed_tweet_cite, is_junk_or_search_url, is_x_status_url, same_publisher_domain,
scrub_https_url,
Expand Down Expand Up @@ -294,12 +295,54 @@ pub async fn require_ship_cite_reachable(
///
/// Returns operator-facing text when the floor is missed.
pub fn require_link_options_floor(link_options: &[String]) -> Result<(), String> {
if link_options.len() < LINK_OPTIONS_MIN {
return Err(format!(
"Need at least {LINK_OPTIONS_MIN} reachable publisher Link options (got {}). \
require_link_options_floor_min(LINK_OPTIONS_MIN, link_options, None)
}

/// Tweet floor: locked X status cite needs only the operator URL; publisher cites need three.
///
/// # Errors
///
/// Returns operator-facing text when the floor is missed.
pub fn require_tweet_link_options_floor(
brief: &str,
link_options: &[String],
) -> Result<(), String> {
let min = tweet_link_options_min(brief);
let hint = if min == 1 {
Some(
"Operator locked an X status cite; Link:1 is the quote card. \
Add a publisher https in the brief for extra Link options.",
)
} else {
None
};
require_link_options_floor_min(min, link_options, hint)
}

fn tweet_link_options_min(brief: &str) -> usize {
if extract_brief_cite(brief).is_some_and(|u| is_x_status_url(&u)) {
1
} else {
LINK_OPTIONS_MIN
}
}

fn require_link_options_floor_min(
min: usize,
link_options: &[String],
hint: Option<&str>,
) -> Result<(), String> {
if link_options.len() < min {
let mut msg = format!(
"Need at least {min} reachable publisher Link options (got {}). \
Refuse draft/tweet with Link:0. Retry with a live publisher URL, or `/draft_about` / `/tweet_about` with a cite.",
link_options.len()
));
);
if let Some(h) = hint {
msg.push(' ');
msg.push_str(h);
}
return Err(msg);
}
if link_options
.iter()
Expand Down Expand Up @@ -504,6 +547,17 @@ mod tests {
assert!(require_link_options_floor(&three[..2]).is_err());
}

#[test]
fn tweet_link_floor_one_when_operator_locked_x_status_cite() {
let x = "https://x.com/nineshoot/status/2094567713113059575";
let brief = format!("Obscura Rust browser, Short punchy take, Link cite {x}");
let one = vec![x.to_string()];
assert!(require_tweet_link_options_floor(&brief, &one).is_ok());
assert!(require_link_options_floor(&one).is_err());
let publisher_brief = "Obscura Rust browser, cite https://labs.sogeti.com/obscura";
assert!(require_tweet_link_options_floor(publisher_brief, &one).is_err());
}

#[test]
fn propose_serp_junk_cannot_satisfy_link_floor() {
let propose =
Expand Down
21 changes: 19 additions & 2 deletions backend/crates/itcy/src/sources/tweet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ pub async fn build_grounded_tweet(
}
}
let (body, link_options) = attach_tweet_cites(&tweet_body, &pack_for_links, &tweet_id, subject);
crate::sources::publisher_url::require_link_options_floor(&link_options)
crate::sources::publisher_url::require_tweet_link_options_floor(subject, &link_options)
.map_err(RagError::Store)?;
info!(
tweet_id = %tweet_id,
Expand Down Expand Up @@ -204,7 +204,7 @@ pub async fn build_grounded_tweet_from_pack(
.await;
let tweet_body = ensure_tweet_handles_from_pack(tools, &tweet_body, &research_pack);
let (body, link_options) = attach_tweet_cites(&tweet_body, &urls, &tweet_id, subject);
crate::sources::publisher_url::require_link_options_floor(&link_options)
crate::sources::publisher_url::require_tweet_link_options_floor(subject, &link_options)
.map_err(RagError::Store)?;
Ok(GroundedDraft {
subject: subject.to_string(),
Expand Down Expand Up @@ -480,6 +480,23 @@ Sources:
assert!(api.contains("tinyboot"));
}

#[test]
fn locked_x_status_cite_passes_link_floor_with_one_option() {
let x = "https://x.com/nineshoot/status/2094567713113059575";
let brief = format!("Obscura Rust browser, cite {x}");
let pack = vec![x.to_string()];
let (_out, opts) = attach_tweet_cites(
"🚀 Obscura ships Rust + V8 + CDP.\n\n#Rust #Browser\n",
&pack,
"TWEET-098",
&brief,
);
crate::sources::publisher_url::require_tweet_link_options_floor(&brief, &opts)
.expect("locked X cite with one Link option");
assert_eq!(opts.len(), 1);
assert_eq!(opts[0], x);
}

#[test]
fn x_status_cite_keeps_at_least_three_link_options_from_pack() {
let x = "https://x.com/a/status/1";
Expand Down
51 changes: 36 additions & 15 deletions backend/crates/itcy/src/sources/tweet_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ use std::fmt::Write;
use tracing::{info, warn};

const PACK_CAP: usize = crate::sources::publisher_url::LINK_OPTIONS_CAP;
/// Extra X statuses to pad Link options when the cite is a locked X status (no SERP).
const LINK_OPTIONS_PAD: usize = crate::sources::publisher_url::LINK_OPTIONS_MIN - 1;
/// Pack summary budget for the browsed cite page. X accessibility trees are verbose; the
/// JPEG XL card `/url:` sat past 2k (~3.5k). Publisher URLs are still taken from the **full**
/// browse before this clip.
Expand Down Expand Up @@ -58,11 +60,14 @@ pub async fn run_short_cite_load(
};

crate::sources::rag::log_pipeline_step("4/4 X search");
let x_q = x_search_query(subject);
let (x_extra, x_hits) = if x_cite_only {
info!("load_tweet: subject is X status; skip X keyword search");
(None, 0)
info!(
query = %x_q,
"load_tweet: X query (pad Link options without SERP)"
);
extra_x_statuses(&x_q, subject_url, LINK_OPTIONS_PAD).await
} else {
let x_q = x_search_query(subject);
info!(query = %x_q, "load_tweet: X query");
let pair = extra_x_status(&x_q, subject_url).await;
info!(
Expand All @@ -71,7 +76,7 @@ pub async fn run_short_cite_load(
picked = pair.0.as_deref().unwrap_or("(none)"),
"load_tweet: X results"
);
pair
(pair.0.into_iter().collect::<Vec<_>>(), pair.1)
};

let session = session_publisher_urls(tools, x_cite_only).await;
Expand Down Expand Up @@ -150,7 +155,7 @@ struct PackAssemble<'a> {
cite_text: &'a str,
overview: &'a str,
session: &'a [String],
x_extra: Option<String>,
x_extra: Vec<String>,
}

async fn assemble_and_probe_pack(
Expand All @@ -165,9 +170,7 @@ async fn assemble_and_probe_pack(
parts.overview,
parts.session,
);
if let Some(u) = parts.x_extra.clone() {
push_unique(&mut urls, std::iter::once(u));
}
push_unique(&mut urls, parts.x_extra.iter().cloned());
info!(
before_probe = urls.len(),
urls = %urls.iter().take(12).cloned().collect::<Vec<_>>().join(" | "),
Expand Down Expand Up @@ -252,24 +255,42 @@ async fn brave_and_extra_browse(
}

async fn extra_x_status(query: &str, subject_url: &str) -> (Option<String>, usize) {
let (extras, hits) = extra_x_statuses(query, subject_url, 1).await;
(extras.into_iter().next(), hits)
}

async fn extra_x_statuses(query: &str, subject_url: &str, max: usize) -> (Vec<String>, usize) {
if max == 0 {
return (Vec::new(), 0);
}
let Ok(tool) = TwitterTool::from_disk() else {
warn!("load_tweet: X search skipped (no creds)");
return (None, 0);
return (Vec::new(), 0);
};
let hits = match tool.search(&[query.to_string()]).await {
Ok(h) => h,
Err(e) => {
warn!(error = %e, query = %query, "load_tweet: X search failed");
return (None, 0);
return (Vec::new(), 0);
}
};
let n = hits.len();
let subject_id = x_status_id(subject_url);
let picked = hits
.into_iter()
.map(|h| h.url)
.find(|u| is_x_status_url(u) && x_status_id(u) != subject_id);
(picked, n)
let mut extras = Vec::new();
for h in hits {
let u = h.url;
if !is_x_status_url(&u) || x_status_id(&u) == subject_id {
continue;
}
if extras.iter().any(|x| x == &u) {
continue;
}
extras.push(u);
if extras.len() >= max {
break;
}
}
(extras, n)
}

fn first_extra_publisher(extracted: &[String], subject_url: &str) -> Option<String> {
Expand Down
Loading