pub const PAIR: u8 = 0x01;
pub const END: u8 = 0x00;
/// a varint holds 63 bits, 7 per byte
const VARINT_MAX_BYTES: u32 = 9;
const SHORT_KEY: usize = 64;
pub type Pair = (Vec<u8>, Vec<u8>);
#[derive(Debug, thiserror::Error)]
out.write_all(&meta)?;
Ok(())
}
/// a short key goes out in one write with the control byte and both lengths, because a zstd writer
/// pays for every call
pub fn write_pair(out: &mut impl Write, key: &[u8], value: &[u8]) -> io::Result<()> {
if key.len() > SHORT_KEY {
out.write_all(&[PAIR])?;
write_varint(out, key.len() as u64)?;
out.write_all(key)?;
write_varint(out, value.len() as u64)?;
out.write_all(&[PAIR])?;
write_varint(out, key.len() as u64)?;
out.write_all(key)?;
write_varint(out, value.len() as u64)?;
return out.write_all(value);
}
let mut head = [0; 1 + 2 * VARINT_MAX_BYTES as usize + SHORT_KEY];
head[0] = PAIR;
let mut at = 1;
at += put_varint(&mut head[at..], key.len() as u64);
head[at..at + key.len()].copy_from_slice(key);
at += key.len();
at += put_varint(&mut head[at..], value.len() as u64);
out.write_all(&head[..at])?;
out.write_all(value)
}
/// nothing may be written after it
pub fn write_end(out: &mut impl Write) -> io::Result<()> {
out.write_all(&[END])
}
/// a slice can't be longer than `isize::MAX`, which fits in the 63 bits a varint has
fn write_varint(out: &mut impl Write, mut n: u64) -> io::Result<()> {
fn write_varint(out: &mut impl Write, n: u64) -> io::Result<()> {
let mut buf = [0; VARINT_MAX_BYTES as usize];
let len = put_varint(&mut buf, n);
out.write_all(&buf[..len])
}
/// lengths are at most `isize::MAX`, which fits in the 63 bits a varint holds
fn put_varint(buf: &mut [u8], mut n: u64) -> usize {
let mut len = 0;
while n >= 0x80 {
buf[len] = n as u8 | 0x80;
n >>= 7;
len += 1;
}
buf[len] = n as u8;
out.write_all(&buf[..=len])
len + 1
}
/// yields the pairs in archive order. the first error ends it, and the pairs before an error are
/// from an archive that isn't valid, so a caller that can't take them back should collect first
164 unmodified lines
Expand all
kind: "a.b.c",
stream: "s",
first: 300,
};
// keys either side of `SHORT_KEY`, values either side of the 127/128 varint boundary
let pairs = [
(b"b".to_vec(), vec![1; 200]),
(vec![], vec![]),
(vec![2; SHORT_KEY], vec![3; 127]),
(vec![4; SHORT_KEY + 1], vec![5; 128]),
(b"b".to_vec(), b"x".to_vec()),
];
write_header(&mut star, &meta).unwrap();
write_pair(&mut star, b"b", &[0; 200]).unwrap();
write_pair(&mut star, b"", b"").unwrap();
write_pair(&mut star, b"b", b"x").unwrap();
for (key, value) in &pairs {
write_pair(&mut star, key, value).unwrap();
}
write_end(&mut star).unwrap();
let reader = Reader::new(&star[..]).unwrap();
assert_eq!(reader.meta(), serde_ipld_dagcbor::to_vec(&meta).unwrap());
let pairs: Vec<_> = reader.collect::<Result<_, _>>().unwrap();
assert_eq!(
pairs,
[
(b"b".to_vec(), vec![0; 200]),
(vec![], vec![]),
(b"b".to_vec(), b"x".to_vec()),
]
);
let read: Vec<_> = reader.collect::<Result<_, _>>().unwrap();
assert_eq!(read, pairs);
}
#[test]
fn a_header_without_a_text_type_is_not_written() {