@@ -1,8 +1,10 @@ use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; use anyhow::Result; use chrono::{SecondsFormat, TimeZone, Utc}; +use regex::Regex; use crate::memory::{to_base36, MemoryLane, MemoryStore, ResolvedRef}; #[derive( @@ -189,9 +191,66 @@ pub lanes: Vec, pub estimated_tokens: usize, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidenceValueKind { + Number, + Currency, + Percent, + Duration, + Date, + Time, + Ordinal, +} + +impl EvidenceValueKind { + pub fn as_str(&self) -> &'static str { + match self { + Self::Number => "number", + Self::Currency => "currency", + Self::Percent => "percent", + Self::Duration => "duration", + Self::Date => "date", + Self::Time => "time", + Self::Ordinal => "ordinal", + } + } +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct EvidenceValueMention { + pub text: String, + pub number: Option, + pub kind: EvidenceValueKind, + pub unit: Option, +} + +impl EvidenceValueMention { + pub(crate) fn aggregate_signature(&self) -> Option { + let number = self.number?; + if !number.is_finite() { + return None; + } + if !matches!( + self.kind, + EvidenceValueKind::Number + | EvidenceValueKind::Currency + | EvidenceValueKind::Percent + | EvidenceValueKind::Duration + ) { + return None; + } + Some(format!( + "{}:{}", + self.kind.as_str(), + self.unit.as_deref().unwrap_or("") + )) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct EvidenceFactRow { pub row_id: String, pub ref_id: String, pub entity_type: String, @@ -199,8 +258,10 @@ pub timestamp: Option, pub ordinal: usize, pub value_text: Option, pub value_number: Option, + #[serde(default)] + pub values: Vec, pub text: String, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -438,9 +499,10 @@ .memory .ref_session_id(ref_id)? .or_else(|| fallback_session_id.map(str::to_string)); let timestamp = self.memory.ref_timestamp(ref_id)?; - let (value_text, value_number) = first_numeric_value(body); + let values = numeric_value_mentions(body); + let (value_text, value_number) = primary_numeric_value(&values); let text = truncate_chars(body, 180); let row_seed = format!("{ref_id}:{ordinal}:{}", text); Ok(EvidenceFactRow { row_id: format!("fact_{}", stable_base36(&row_seed)), @@ -450,8 +512,9 @@ timestamp, ordinal, value_text, value_number, + values, text, }) }) .collect() @@ -571,8 +634,14 @@ } if let Some(value) = fact.value_number { attrs.push_str(&format!(" number=\"{}\"", value)); } + if !fact.values.is_empty() { + attrs.push_str(&format!( + " values=\"{}\"", + xml_escape(&render_value_mentions(&fact.values)) + )); + } let line = format!( " {}\n", attrs, xml_escape(&truncate_chars(&fact.text, 120)), @@ -1093,39 +1162,324 @@ .collect() } fn first_numeric_value(text: &str) -> (Option, Option) { - let mut current = String::new(); - let mut has_digit = false; - for ch in text.chars() { - if ch.is_ascii_digit() - || (has_digit && matches!(ch, '.' | ',' | ':' | '/' | '-')) - || (!has_digit && matches!(ch, '+' | '-')) - { - if ch.is_ascii_digit() { - has_digit = true; + primary_numeric_value(&numeric_value_mentions(text)) +} + +fn primary_numeric_value(values: &[EvidenceValueMention]) -> (Option, Option) { + values + .iter() + .find(|value| value.aggregate_signature().is_some()) + .or_else(|| values.first()) + .map(|value| (Some(value.text.clone()), value.number)) + .unwrap_or((None, None)) +} + +fn numeric_value_mentions(text: &str) -> Vec { + numeric_token_regex() + .find_iter(text) + .filter(|mat| !token_embedded_in_word(text, mat.start(), mat.end())) + .map(|mat| { + let token = mat.as_str(); + let token_lower = token.to_ascii_lowercase(); + let next_word = next_word_after(text, mat.end()); + let previous_word = previous_word_before(text, mat.start()); + let (kind, unit, mention_end) = classify_numeric_mention( + text, + mat.start(), + mat.end(), + token, + &token_lower, + previous_word.as_deref(), + next_word.as_ref(), + ); + let text = text[mat.start()..mention_end].trim().to_string(); + let number = parse_mention_number(token, &kind); + EvidenceValueMention { + text, + number, + kind, + unit, } - current.push(ch); - } else if has_digit { + }) + .collect() +} + +fn numeric_token_regex() -> &'static Regex { + static NUMERIC_TOKEN_RE: OnceLock = OnceLock::new(); + NUMERIC_TOKEN_RE.get_or_init(|| { + Regex::new( + r"\$?[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?(?:[:/-]\d{1,4})*(?:%|st|nd|rd|th)?", + ) + .expect("numeric value regex compiles") + }) +} + +fn classify_numeric_mention( + text: &str, + start: usize, + end: usize, + token: &str, + token_lower: &str, + previous_word: Option<&str>, + next_word: Option<&(String, usize)>, +) -> (EvidenceValueKind, Option, usize) { + let next_word_text = next_word.map(|(word, _)| word.as_str()); + let next_word_end = next_word.map(|(_, end)| *end).unwrap_or(end); + + if is_list_marker(text, start, end) { + return (EvidenceValueKind::Ordinal, None, end); + } + if is_currency_mention(token, previous_word, next_word_text) { + let mention_end = next_word_text + .filter(|word| is_currency_word(word)) + .map(|_| next_word_end) + .unwrap_or(end); + return ( + EvidenceValueKind::Currency, + Some("$".to_string()), + mention_end, + ); + } + if is_percent_mention(token_lower, next_word_text) { + let mention_end = next_word_text + .filter(|word| matches!(*word, "percent" | "percentage")) + .map(|_| next_word_end) + .unwrap_or(end); + return ( + EvidenceValueKind::Percent, + Some("%".to_string()), + mention_end, + ); + } + if token.contains(':') { + return (EvidenceValueKind::Time, None, end); + } + if is_date_like(token, token_lower, previous_word, next_word_text) { + return (EvidenceValueKind::Date, None, end); + } + if has_ordinal_suffix(token_lower) { + return (EvidenceValueKind::Ordinal, None, end); + } + if let Some(unit) = next_word_text.and_then(duration_unit) { + return (EvidenceValueKind::Duration, Some(unit), next_word_end); + } + (EvidenceValueKind::Number, None, end) +} + +fn token_embedded_in_word(text: &str, start: usize, end: usize) -> bool { + let before = text[..start].chars().next_back(); + let after = text[end..].chars().next(); + let before_word = before.is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_'); + let after_word = after.is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_'); + before_word || after_word +} + +fn next_word_after(text: &str, index: usize) -> Option<(String, usize)> { + let mut word_start = None; + for (offset, ch) in text[index..].char_indices() { + if ch.is_whitespace() || ch == '-' { + continue; + } + if ch.is_ascii_alphabetic() { + word_start = Some(index + offset); + } + break; + } + let start = word_start?; + let mut end = start; + for (offset, ch) in text[start..].char_indices() { + if ch.is_ascii_alphabetic() { + end = start + offset + ch.len_utf8(); + } else { break; + } + } + Some((text[start..end].to_ascii_lowercase(), end)) +} + +fn previous_word_before(text: &str, index: usize) -> Option { + let prefix = &text[..index]; + let mut end = prefix.len(); + while let Some((pos, ch)) = prefix[..end].char_indices().next_back() { + if ch.is_whitespace() || ch == '-' { + end = pos; } else { - current.clear(); + break; } } - if !has_digit { - return (None, None); + let mut start = end; + while let Some((pos, ch)) = prefix[..start].char_indices().next_back() { + if ch.is_ascii_alphabetic() { + start = pos; + } else { + break; + } } - let value_text = current - .trim_matches(|ch: char| !ch.is_ascii_digit()) - .to_string(); - if value_text.is_empty() { - return (None, None); + (start < end).then(|| prefix[start..end].to_ascii_lowercase()) +} + +fn is_list_marker(text: &str, start: usize, end: usize) -> bool { + let Some(punct) = text[end..].chars().next() else { + return false; + }; + if !matches!(punct, '.' | ')') { + return false; } - let normalized = value_text.replace(',', ""); - let value_number = normalized.parse::().ok(); - (Some(value_text), value_number) + let after_punct = end + punct.len_utf8(); + if text[after_punct..] + .chars() + .next() + .is_some_and(|ch| !ch.is_whitespace()) + { + return false; + } + let line_start = text[..start] + .rfind('\n') + .map(|index| index + 1) + .unwrap_or(0); + let prefix = text[line_start..start].trim(); + prefix.is_empty() || matches!(prefix, "-" | "*" | "+") } +fn is_currency_mention(token: &str, previous_word: Option<&str>, next_word: Option<&str>) -> bool { + token.starts_with('$') + || previous_word.is_some_and(is_currency_word) + || next_word.is_some_and(is_currency_word) +} + +fn is_currency_word(word: &str) -> bool { + matches!(word, "dollar" | "dollars" | "usd" | "buck" | "bucks") +} + +fn is_percent_mention(token_lower: &str, next_word: Option<&str>) -> bool { + token_lower.ends_with('%') + || next_word.is_some_and(|word| matches!(word, "percent" | "percentage")) +} + +fn is_date_like( + token: &str, + token_lower: &str, + previous_word: Option<&str>, + next_word: Option<&str>, +) -> bool { + if token.contains('/') || token[1.min(token.len())..].contains('-') { + return true; + } + if previous_word.is_some_and(is_month_word) || next_word.is_some_and(is_month_word) { + return true; + } + let normalized = strip_numeric_affixes(token_lower); + let is_year = normalized + .parse::() + .is_ok_and(|year| (1900..=2100).contains(&year)); + is_year + && (previous_word.is_some_and(is_date_context_word) + || next_word.is_some_and(is_date_context_word)) +} + +fn is_month_word(word: &str) -> bool { + matches!( + word, + "january" + | "jan" + | "february" + | "feb" + | "march" + | "mar" + | "april" + | "apr" + | "may" + | "june" + | "jun" + | "july" + | "jul" + | "august" + | "aug" + | "september" + | "sep" + | "sept" + | "october" + | "oct" + | "november" + | "nov" + | "december" + | "dec" + ) +} + +fn is_date_context_word(word: &str) -> bool { + is_month_word(word) || matches!(word, "date" | "year" | "birthday" | "anniversary") +} + +fn has_ordinal_suffix(token_lower: &str) -> bool { + ["st", "nd", "rd", "th"].iter().any(|suffix| { + token_lower + .strip_suffix(suffix) + .is_some_and(ends_with_digit) + }) +} + +fn duration_unit(word: &str) -> Option { + let unit = match word { + "day" | "days" => "day", + "week" | "weeks" => "week", + "month" | "months" => "month", + "year" | "years" => "year", + "hour" | "hours" | "hr" | "hrs" => "hour", + "minute" | "minutes" | "min" | "mins" => "minute", + "second" | "seconds" | "sec" | "secs" => "second", + "night" | "nights" => "night", + _ => return None, + }; + Some(unit.to_string()) +} + +fn parse_mention_number(token: &str, kind: &EvidenceValueKind) -> Option { + if matches!( + kind, + EvidenceValueKind::Date | EvidenceValueKind::Time | EvidenceValueKind::Ordinal + ) { + return None; + } + let normalized = strip_numeric_affixes(&token.to_ascii_lowercase()).replace(',', ""); + normalized.parse::().ok() +} + +fn strip_numeric_affixes(token: &str) -> String { + let token = token.trim_start_matches('$').trim_end_matches('%'); + ["st", "nd", "rd", "th"] + .iter() + .find_map(|suffix| token.strip_suffix(suffix)) + .unwrap_or(token) + .to_string() +} + +fn ends_with_digit(value: &str) -> bool { + value + .chars() + .next_back() + .is_some_and(|ch| ch.is_ascii_digit()) +} + +fn render_value_mentions(values: &[EvidenceValueMention]) -> String { + values + .iter() + .take(8) + .map(|value| { + let mut rendered = format!("{}:{}", value.kind.as_str(), value.text); + if let Some(unit) = &value.unit { + rendered.push('['); + rendered.push_str(unit); + rendered.push(']'); + } + rendered + }) + .collect::>() + .join(";") +} + pub(crate) fn stable_base36(value: &str) -> String { let mut hash: u64 = 1469598103934665603; for byte in value.as_bytes() { hash ^= *byte as u64; @@ -1207,8 +1561,14 @@ timestamp: Some(42), ordinal: 0, value_text: Some("27:45".to_string()), value_number: None, + values: vec![EvidenceValueMention { + text: "27:45".to_string(), + number: None, + kind: EvidenceValueKind::Time, + unit: None, + }], text: "personal best was 27:45".to_string(), }]; packet.timeline = vec![EvidenceTimelineEvent { ref_id: "ref_fact".to_string(), @@ -1224,12 +1584,39 @@ assert!(rendered.contains("")); assert!(rendered.contains("ref=\"ref_fact\"")); assert!(rendered.contains("date=\"1970-01-01T00:00:42Z\"")); assert!(rendered.contains("value=\"27:45\"")); + assert!(rendered.contains("values=\"time:27:45\"")); assert!(rendered.contains(">(), + vec![ + EvidenceValueKind::Ordinal, + EvidenceValueKind::Currency, + EvidenceValueKind::Duration, + EvidenceValueKind::Date, + ] + ); + assert_eq!(values[1].text, "$40"); + assert_eq!(values[1].number, Some(40.0)); + assert_eq!(values[1].unit.as_deref(), Some("$")); + assert_eq!(values[2].text, "3 days"); + assert_eq!(values[2].number, Some(3.0)); + assert_eq!(values[2].unit.as_deref(), Some("day")); + assert_eq!( + first_numeric_value(text), + (Some("$40".to_string()), Some(40.0)) + ); + } + + #[test] fn rendered_packet_with_refs_tracks_only_emitted_evidence_refs() { let mut packet = test_packet_with_refs( "pkt_rendered_refs", &["ref_anchor", "ref_omitted"], @@ -1280,8 +1667,9 @@ timestamp: Some(42 + ordinal as i64), ordinal, value_text: Some(format!("{ordinal}")), value_number: Some(ordinal as f64), + values: vec![], text: "fact row with enough text to consume visible packet budget".repeat(3), }) .collect(); packet.timeline = packet @@ -1346,8 +1734,9 @@ timestamp: Some(42 + ordinal as i64), ordinal, value_text: None, value_number: None, + values: vec![], text: packet.bodies[ordinal].clone(), }) .collect(); packet.timeline = packet @@ -1398,8 +1787,9 @@ timestamp: Some(42 + ordinal as i64), ordinal, value_text: None, value_number: None, + values: vec![], text: packet.bodies[ordinal].clone(), }) .collect();