list pages serialize each record once
Signed-off-by: dawn <90008@klbr.net>
23 unmodified lines24252627282829303132333334353637383639404142434445464347484950512 unmodified lines54555657545556575858596061626364656610 unmodified lines777879807781828384851 unmodified line8788899091929394959697989910010110210310410510610768 unmodified lines23 unmodified lines pool: &mut Pool, repo: &ChainState,) -> anyhow::Result<()> { let name = repo.dir_name(); let mut collections: BTreeMap<&str, Vec<&ServedRecord>> = BTreeMap::new(); let mut collections: BTreeMap<&str, Vec<Listed>> = BTreeMap::new(); for record in &repo.records { collections .entry(&record.collection) .or_default() .push(record); .push(Listed { record, answer: serde_json::value::to_raw_value(&record.answer)?, }); } for (collection, mut records) in collections { records.sort_by(|a, b| a.rkey.cmp(&b.rkey)); records.sort_by(|a, b| a.record.rkey.cmp(&b.record.rkey)); let records: Vec<&Listed> = records.iter().collect(); for &(label, _) in RECORD_DIRECTIONS { let mut ordered = records.clone(); if label == FORWARD { ordered.reverse(); } for &size in RECORD_SIZES { let mut pages: Vec<Vec<&ServedRecord>> = let mut pages: Vec<Vec<&Listed>> = records.chunks(size).map(|chunk| chunk.to_vec()).collect(); if label == FORWARD { pages.reverse(); for chunk in &mut pages {2 unmodified lines } let mut prior_last: Option<&str> = None; for (n, chunk) in pages.iter().enumerate() { let more = n + 1 < pages.len(); let cursor = chunk.last().map(|record| record.rkey.as_str()); let body = json!({ "records": chunk.iter().map(|record| &record.answer).collect::<Vec<_>>(), "cursor": cursor.filter(|_| more), }); let cursor = chunk.last().map(|listed| listed.record.rkey.as_str()); let body = Page { cursor: cursor.filter(|_| more), records: chunk.iter().map(|listed| &*listed.answer).collect(), }; let file = put_page(pool, &name, "lists", &body)?; let after = match prior_last { None => vec![String::new()], Some(prev) => spellings(prev),10 unmodified lines } } } if let Some(last) = ordered.last() { for cursor in spellings(&last.rkey) { for cursor in spellings(&last.record.rkey) { let key = format!("{name}|{label}|{collection}|{cursor}"); list_one.push((key, EMPTY_RECORDS.into())); } }1 unmodified line } Ok(())}
/// each record is serialized once, not once per page size and directionstruct Listed<'a> { record: &'a ServedRecord, answer: Box<serde_json::value::RawValue>,}
/// the fields in the order a `json!` map writes them, so the pages keep their names#[derive(serde::Serialize)]struct Page<'a> { cursor: Option<&'a str>, records: Vec<&'a serde_json::value::RawValue>,}
pub(crate) fn maps( record_page: Vec<(String, String)>, record_dir: Vec<(String, String)>, list: Vec<(String, String)>,68 unmodified lines233 unmodified linesExpand all234235236237238238239240241242112 unmodified linesExpand all233 unmodified linesExpand allpub(crate) fn put_page( pool: &mut Pool, repo_dir_name: &str, subdir: &str, body: &serde_json::Value, body: &impl Serialize,) -> anyhow::Result<String> { let bytes = serde_json::to_vec(body)?; let file = format!("{subdir}/{}.json", short_hash(&bytes)); pool.put(&file, &bytes)?;112 unmodified linesExpand all281 unmodified linesExpand all282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321134 unmodified linesExpand all281 unmodified linesExpand all );}
#[test]fn a_list_page_is_named_by_the_bytes_its_json_map_writes() { let mut repo = repo(120, 1); for (n, record) in repo.records.iter_mut().enumerate() { record.answer["value"] = json!({ "text": format!("é \"quoted\" \\ \u{1}{n} 🐱"), "nested": { "z": [n, null, true, 1.5], "a": "" }, }); } let name = repo.dir_name(); let emitted = emitted(&repo); let dir = emitted.root.join(&name).join("lists"); let mut pages = 0; for entry in std::fs::read_dir(&dir).unwrap() { let path = entry.unwrap().path(); let bytes = std::fs::read(&path).unwrap(); let page: Value = serde_json::from_slice(&bytes).unwrap(); let written = serde_json::to_vec(&json!({ "records": page["records"], "cursor": page["cursor"], })) .unwrap(); assert!(bytes == written, "{}", path.display()); assert_eq!( path.file_stem().unwrap().to_str().unwrap(), short_hash(&bytes) ); pages += 1; } assert!(pages > 100, "{pages}");}
#[test]fn cursors_are_spelled_both_ways_and_unknown_cursors_miss() { let mut repo = repo(60, 1); repo.records[50].rkey = "r050:b".into(); let name = repo.dir_name();134 unmodified linesExpand all