diff --git a/src/cache/check.rs b/src/cache/check.rs index 261840b..01882e2 100644 --- a/src/cache/check.rs +++ b/src/cache/check.rs @@ -8,6 +8,7 @@ use crate::cache::hint::*; use crate::cache::mapping::*; use crate::cache::superblock::*; use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; +use crate::pdata::array; use crate::pdata::array_walker::*; use crate::pdata::bitset_walker::*; @@ -33,30 +34,30 @@ mod format1 { } } - fn check_flags(&self, m: &Mapping) -> anyhow::Result<()> { + fn check_flags(&self, m: &Mapping) -> array::Result<()> { if (m.flags & !(MappingFlags::Valid as u32 | MappingFlags::Dirty as u32)) != 0 { - return Err(anyhow!("unknown flags in mapping: {}", m.flags)); + return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags))); } if !m.is_valid() && m.is_dirty() { - return Err(anyhow!("dirty bit found on an unmapped block")); + return Err(array::value_err(format!("dirty bit found on an unmapped block"))); } Ok(()) } - fn check_oblock(&self, m: &Mapping) -> anyhow::Result<()> { + fn check_oblock(&self, m: &Mapping) -> array::Result<()> { if !m.is_valid() { if m.oblock > 0 { - return Err(anyhow!("invalid mapped block")); + return Err(array::value_err(format!("invalid block is mapped"))); } return Ok(()); } if m.oblock >= self.nr_origin_blocks { - return Err(anyhow!("mapping beyond end of the origin device")); + return Err(array::value_err(format!("mapping beyond end of the origin device"))); } let mut seen_oblocks = self.seen_oblocks.lock().unwrap(); if seen_oblocks.contains(&m.oblock) { - return Err(anyhow!("origin block already mapped")); + return Err(array::value_err(format!("origin block already mapped"))); } seen_oblocks.insert(m.oblock); @@ -65,7 +66,7 @@ mod format1 { } impl ArrayVisitor for MappingChecker { - fn visit(&self, _index: u64, m: Mapping) -> anyhow::Result<()> { + fn visit(&self, _index: u64, m: Mapping) -> array::Result<()> { self.check_flags(&m)?; self.check_oblock(&m)?; @@ -98,28 +99,28 @@ mod format2 { } } - fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> anyhow::Result<()> { + fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> array::Result<()> { if (m.flags & !(MappingFlags::Valid as u32)) != 0 { - return Err(anyhow!("unknown flags in mapping: {}", m.flags)); + return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags))); } if !m.is_valid() && dirty_bit { - return Err(anyhow!("dirty bit found on an unmapped block")); + return Err(array::value_err(format!("dirty bit found on an unmapped block"))); } Ok(()) } - fn check_oblock(&self, m: &Mapping, seen_oblocks: &mut BTreeSet) -> anyhow::Result<()> { + fn check_oblock(&self, m: &Mapping, seen_oblocks: &mut BTreeSet) -> array::Result<()> { if !m.is_valid() { if m.oblock > 0 { - return Err(anyhow!("invalid mapped block")); + return Err(array::value_err(format!("invalid mapped block"))); } return Ok(()); } if m.oblock >= self.nr_origin_blocks { - return Err(anyhow!("mapping beyond end of the origin device")); + return Err(array::value_err(format!("mapping beyond end of the origin device"))); } if seen_oblocks.contains(&m.oblock) { - return Err(anyhow!("origin block already mapped")); + return Err(array::value_err(format!("origin block already mapped"))); } seen_oblocks.insert(m.oblock); @@ -128,7 +129,7 @@ mod format2 { } impl ArrayVisitor for MappingChecker { - fn visit(&self, index: u64, m: Mapping) -> anyhow::Result<()> { + fn visit(&self, index: u64, m: Mapping) -> array::Result<()> { let mut inner = self.inner.lock().unwrap(); self.check_flags(&m, inner.dirty_bits.contains(index as usize))?; self.check_oblock(&m, &mut inner.seen_oblocks)?; @@ -149,7 +150,7 @@ impl HintChecker { } impl ArrayVisitor for HintChecker { - fn visit(&self, _index: u64, _hint: Hint) -> anyhow::Result<()> { + fn visit(&self, _index: u64, _hint: Hint) -> array::Result<()> { // TODO: check hints Ok(()) } diff --git a/src/cache/dump.rs b/src/cache/dump.rs index eacda2a..9a21eed 100644 --- a/src/cache/dump.rs +++ b/src/cache/dump.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Result}; +use anyhow::anyhow; use fixedbitset::FixedBitSet; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -8,6 +8,7 @@ use crate::cache::mapping::Mapping; use crate::cache::superblock::*; use crate::cache::xml::{self, MetadataVisitor}; use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; +use crate::pdata::array; use crate::pdata::array_walker::*; //------------------------------------------ @@ -45,7 +46,7 @@ mod format1 { } impl<'a> ArrayVisitor for MappingEmitter<'a> { - fn visit(&self, index: u64, m: Mapping) -> Result<()> { + fn visit(&self, index: u64, m: Mapping) -> array::Result<()> { if m.is_valid() { let m = xml::Map { cblock: index as u32, @@ -55,7 +56,7 @@ mod format1 { let mut inner = self.inner.lock().unwrap(); inner.valid_mappings.set(index as usize, true); - inner.visitor.mapping(&m)?; + inner.visitor.mapping(&m).map_err(|e| array::value_err(format!("{}", e)))?; } Ok(()) @@ -89,7 +90,7 @@ mod format2 { } impl ArrayVisitor for DirtyVisitor { - fn visit(&self, index: u64, bits: u64) -> Result<()> { + fn visit(&self, index: u64, bits: u64) -> array::Result<()> { for i in 0..64u64 { if (index + i) >= self.nr_entries as u64 { break; @@ -132,7 +133,7 @@ mod format2 { } impl<'a> ArrayVisitor for MappingEmitter<'a> { - fn visit(&self, index: u64, m: Mapping) -> Result<()> { + fn visit(&self, index: u64, m: Mapping) -> array::Result<()> { if m.is_valid() { let mut inner = self.inner.lock().unwrap(); let dirty = inner.dirty_bits.contains(index as usize); @@ -143,7 +144,7 @@ mod format2 { }; inner.valid_mappings.set(index as usize, true); - inner.visitor.mapping(&m)?; + inner.visitor.mapping(&m).map_err(|e| array::value_err(format!("{}", e)))?; } Ok(()) @@ -168,14 +169,18 @@ impl<'a> HintEmitter<'a> { } impl<'a> ArrayVisitor for HintEmitter<'a> { - fn visit(&self, index: u64, hint: Hint) -> anyhow::Result<()> { + fn visit(&self, index: u64, hint: Hint) -> array::Result<()> { if self.valid_mappings.contains(index as usize) { let h = xml::Hint { cblock: index as u32, data: hint.hint.to_vec(), }; - self.emitter.lock().unwrap().hint(&h)?; + self.emitter + .lock() + .unwrap() + .hint(&h) + .map_err(|e| array::value_err(format!("{}", e)))?; } Ok(()) diff --git a/src/checksum.rs b/src/checksum.rs index 0ffbd47..9ffbd70 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -11,6 +11,7 @@ const SUPERBLOCK_CSUM_XOR: u32 = 160774; const BITMAP_CSUM_XOR: u32 = 240779; const INDEX_CSUM_XOR: u32 = 160478; const BTREE_CSUM_XOR: u32 = 121107; +const ARRAY_CSUM_XOR: u32 = 595846735; fn checksum(buf: &[u8]) -> u32 { crc32c(&buf[4..]) ^ 0xffffffff @@ -22,6 +23,7 @@ pub enum BT { NODE, INDEX, BITMAP, + ARRAY, UNKNOWN, } @@ -41,6 +43,7 @@ pub fn metadata_block_type(buf: &[u8]) -> BT { BTREE_CSUM_XOR => BT::NODE, BITMAP_CSUM_XOR => BT::BITMAP, INDEX_CSUM_XOR => BT::INDEX, + ARRAY_CSUM_XOR => BT::ARRAY, _ => BT::UNKNOWN, } } @@ -56,6 +59,7 @@ pub fn write_checksum(buf: &mut [u8], kind: BT) -> Result<()> { NODE => BTREE_CSUM_XOR, BITMAP => BITMAP_CSUM_XOR, INDEX => INDEX_CSUM_XOR, + ARRAY => ARRAY_CSUM_XOR, UNKNOWN => {return Err(anyhow!("Invalid block type"));} }; diff --git a/src/pack/node_encode.rs b/src/pack/node_encode.rs index 466fd40..d146ca3 100644 --- a/src/pack/node_encode.rs +++ b/src/pack/node_encode.rs @@ -114,4 +114,8 @@ pub fn pack_index(w: &mut W, bytes: &[u8]) -> PResult<()> { io_to_pr(pack_literal(w, bytes)) } +pub fn pack_array(w: &mut W, bytes: &[u8]) -> PResult<()> { + io_to_pr(pack_literal(w, bytes)) +} + //------------------------------------- diff --git a/src/pack/toplevel.rs b/src/pack/toplevel.rs index 449de91..98b4dcf 100644 --- a/src/pack/toplevel.rs +++ b/src/pack/toplevel.rs @@ -209,6 +209,7 @@ fn pack_block(w: &mut W, kind: BT, buf: &[u8]) -> Result<()> { BT::NODE => pack_btree_node(w, buf).context("unable to pack btree node")?, BT::INDEX => pack_index(w, buf).context("unable to pack space map index")?, BT::BITMAP => pack_bitmap(w, buf).context("unable to pack space map bitmap")?, + BT::ARRAY => pack_array(w, buf).context("unable to pack array block")?, BT::UNKNOWN => return Err(anyhow!("asked to pack an unknown block type")), } diff --git a/src/pdata/array.rs b/src/pdata/array.rs new file mode 100644 index 0000000..68a04b3 --- /dev/null +++ b/src/pdata/array.rs @@ -0,0 +1,133 @@ +use nom::{multi::count, number::complete::*, IResult}; +use thiserror::Error; + +use crate::io_engine::BLOCK_SIZE; +use crate::pdata::btree; +use crate::pdata::unpack::Unpack; + +//------------------------------------------ + +const ARRAY_BLOCK_HEADER_SIZE: u32 = 24; + +pub struct ArrayBlockHeader { + pub csum: u32, + pub max_entries: u32, + pub nr_entries: u32, + pub value_size: u32, + pub blocknr: u64, +} + +impl Unpack for ArrayBlockHeader { + fn disk_size() -> u32 { + ARRAY_BLOCK_HEADER_SIZE + } + + fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> { + let (i, csum) = le_u32(data)?; + let (i, max_entries) = le_u32(i)?; + let (i, nr_entries) = le_u32(i)?; + let (i, value_size) = le_u32(i)?; + let (i, blocknr) = le_u64(i)?; + + Ok(( + i, + ArrayBlockHeader { + csum, + max_entries, + nr_entries, + value_size, + blocknr, + }, + )) + } +} + +pub struct ArrayBlock { + pub header: ArrayBlockHeader, + pub values: Vec, +} + +//------------------------------------------ + +#[derive(Error, Clone, Debug)] +pub enum ArrayError { + #[error("io_error")] + IoError, + + #[error("block error: {0}")] + BlockError(String), + + #[error("value error: {0}")] + ValueError(String), + + #[error("aggregate: {0:?}")] + Aggregate(Vec), + + #[error("{0:?}, {1}")] + Path(Vec, Box), + + #[error(transparent)] + BTreeError(#[from] btree::BTreeError), +} + +pub fn array_block_err(path: &[u64], msg: &str) -> ArrayError { + ArrayError::Path( + path.to_vec(), + Box::new(ArrayError::BlockError(msg.to_string())), + ) +} + +pub fn value_err(msg: String) -> ArrayError { + ArrayError::ValueError(msg) +} + +pub fn aggregate_error(errs: Vec) -> ArrayError { + ArrayError::Aggregate(errs) +} + +pub type Result = std::result::Result; + +//------------------------------------------ + +fn convert_result<'a, V>(path: &[u64], r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { + r.map_err(|_| array_block_err(path, "parse error")) +} + +pub fn unpack_array_block(path: &[u64], data: &[u8]) -> Result> { + // TODO: collect errors + let (i, header) = + ArrayBlockHeader::unpack(data).map_err(|_| array_block_err( + path, + "Couldn't parse array block header" + ))?; + + // check value_size + if header.value_size != V::disk_size() { + return Err(array_block_err( + path, + &format!( + "value_size mismatch: expected {}, was {}", + V::disk_size(), + header.value_size + ) + )); + } + + // check max_entries + if header.value_size * header.max_entries + ARRAY_BLOCK_HEADER_SIZE > BLOCK_SIZE as u32 { + return Err(array_block_err( + path, + &format!("max_entries is too large ({})", header.max_entries) + )); + } + + // TODO: check nr_entries < max_entries + + // TODO: check block_nr + + let (_i, values) = convert_result(path, count(V::unpack, header.nr_entries as usize)(i))?; + + Ok(ArrayBlock { header, values }) +} + +//------------------------------------------ diff --git a/src/pdata/array_block.rs b/src/pdata/array_block.rs deleted file mode 100644 index b082d92..0000000 --- a/src/pdata/array_block.rs +++ /dev/null @@ -1,60 +0,0 @@ -use anyhow::anyhow; -use anyhow::Result; -use nom::{multi::count, number::complete::*, IResult}; - -use crate::pdata::unpack::Unpack; - -//------------------------------------------ - -pub struct ArrayBlockHeader { - pub csum: u32, - pub max_entries: u32, - pub nr_entries: u32, - pub value_size: u32, - pub blocknr: u64, -} - -impl Unpack for ArrayBlockHeader { - fn disk_size() -> u32 { - 24 - } - - fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> { - let (i, csum) = le_u32(data)?; - let (i, max_entries) = le_u32(i)?; - let (i, nr_entries) = le_u32(i)?; - let (i, value_size) = le_u32(i)?; - let (i, blocknr) = le_u64(i)?; - - Ok(( - i, - ArrayBlockHeader { - csum, - max_entries, - nr_entries, - value_size, - blocknr, - }, - )) - } -} - -pub struct ArrayBlock { - pub header: ArrayBlockHeader, - pub values: Vec, -} - -fn convert_result(r: IResult<&[u8], V>) -> Result<(&[u8], V)> { - r.map_err(|_| anyhow!("parse error")) -} - -pub fn unpack_array_block(data: &[u8]) -> Result> { - // TODO: collect errors - let (i, header) = - ArrayBlockHeader::unpack(data).map_err(|_e| anyhow!("Couldn't parse header"))?; - let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?; - - Ok(ArrayBlock { header, values }) -} - -//------------------------------------------ diff --git a/src/pdata/array_walker.rs b/src/pdata/array_walker.rs index 672259e..e418402 100644 --- a/src/pdata/array_walker.rs +++ b/src/pdata/array_walker.rs @@ -1,8 +1,9 @@ use std::sync::Arc; +use crate::checksum; use crate::io_engine::*; -use crate::pdata::array_block::*; -use crate::pdata::btree::*; +use crate::pdata::array::{self, *}; +use crate::pdata::btree::{self, *}; use crate::pdata::btree_walker::*; use crate::pdata::unpack::*; @@ -15,12 +16,12 @@ pub struct ArrayWalker { // FIXME: define another Result type for array visiting? pub trait ArrayVisitor { - fn visit(&self, index: u64, v: V) -> anyhow::Result<()>; + fn visit(&self, index: u64, v: V) -> array::Result<()>; } struct BlockValueVisitor<'a, V> { engine: Arc, - array_block_visitor: &'a mut dyn ArrayVisitor, + array_visitor: &'a mut dyn ArrayVisitor, } impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> { @@ -30,22 +31,37 @@ impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> { ) -> BlockValueVisitor<'a, V> { BlockValueVisitor { engine: e, - array_block_visitor: v, + array_visitor: v, } } - pub fn visit_array_block(&self, index: u64, array_block: ArrayBlock) { + pub fn visit_array_block(&self, index: u64, array_block: ArrayBlock) -> array::Result<()>{ + let mut errs: Vec = Vec::new(); + let begin = index * array_block.header.max_entries as u64; for i in 0..array_block.header.nr_entries { - self.array_block_visitor - .visit(begin + i as u64, array_block.values[i as usize]) - .unwrap(); + if let Err(e) = self.array_visitor.visit(begin + i as u64, array_block.values[i as usize]) { + errs.push(e); // TODO: add path or keys context? + } + } + + // FIXME: duplicate to BTreeWalker::build_aggregrate() + match errs.len() { + 0 => Ok(()), + 1 => { + let e = errs[0].clone(); + Err(e) + } + _ => { + let e = array::aggregate_error(errs); + Err(e) + } } } } impl<'a, V: Unpack + Copy> NodeVisitor for BlockValueVisitor<'a, V> { - // FIXME: return errors + // FIXME: wrap ArrayError into BTreeError, rather than mapping to value_err? fn visit( &self, path: &[u64], @@ -53,20 +69,54 @@ impl<'a, V: Unpack + Copy> NodeVisitor for BlockValueVisitor<'a, V> { _h: &NodeHeader, keys: &[u64], values: &[u64], - ) -> Result<()> { + ) -> btree::Result<()> { + let mut path = path.to_vec(); + let mut errs: Vec = Vec::new(); + for (n, index) in keys.iter().enumerate() { - let b = self.engine.read(values[n]).map_err(|_| io_err(path))?; - let array_block = unpack_array_block::(b.get_data()).map_err(|_| io_err(path))?; - self.visit_array_block(*index, array_block); + let b = self.engine.read(values[n]).map_err(|_| io_err(&path))?; + + // FIXME: move to unpack_array_block? + let bt = checksum::metadata_block_type(b.get_data()); + if bt != checksum::BT::ARRAY { + errs.push(btree::value_err( + format!("checksum failed for array block {}, {:?}", b.loc, bt) + )); + } + + path.push(values[n]); + match unpack_array_block::(&path, b.get_data()) { + Ok(array_block) => { + if let Err(e) = self.visit_array_block(*index, array_block) { + errs.push(btree::value_err(format!("{}", e))); + } + }, + Err(e) => { + errs.push(btree::value_err(format!("{}", e))); + } + } + path.pop(); } + + // FIXME: duplicate to BTreeWalker::build_aggregrate() + match errs.len() { + 0 => Ok(()), + 1 => { + let e = errs[0].clone(); + Err(e) + } + _ => { + let e = btree::aggregate_error(errs); + Err(e) + } + } + } + + fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> { Ok(()) } - fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> { - Ok(()) - } - - fn end_walk(&self) -> Result<()> { + fn end_walk(&self) -> btree::Result<()> { Ok(()) } } @@ -81,7 +131,7 @@ impl ArrayWalker { } // FIXME: redefine the Result type for array visiting? - pub fn walk(&self, visitor: &mut dyn ArrayVisitor, root: u64) -> Result<()> + pub fn walk(&self, visitor: &mut dyn ArrayVisitor, root: u64) -> array::Result<()> where V: Unpack + Copy, { @@ -89,7 +139,7 @@ impl ArrayWalker { let mut path = Vec::new(); path.push(0); let v = BlockValueVisitor::::new(self.engine.clone(), visitor); - w.walk(&mut path, &v, root) + w.walk(&mut path, &v, root).map_err(|e| ArrayError::BTreeError(e)) } } diff --git a/src/pdata/bitset_walker.rs b/src/pdata/bitset_walker.rs index 89b118d..a75bae9 100644 --- a/src/pdata/bitset_walker.rs +++ b/src/pdata/bitset_walker.rs @@ -1,10 +1,9 @@ -use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use std::sync::{Arc, Mutex}; use crate::io_engine::IoEngine; +use crate::pdata::array; use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker}; -use crate::pdata::btree::{self}; struct BitsetVisitor<'a> { nr_entries: u64, @@ -21,10 +20,10 @@ impl<'a> BitsetVisitor<'a> { } impl<'a> ArrayVisitor for BitsetVisitor<'a> { - fn visit(&self, index: u64, bits: u64) -> Result<()> { + fn visit(&self, index: u64, bits: u64) -> array::Result<()> { let begin: u64 = index << 6; if begin > self.nr_entries { - return Err(anyhow!("bitset size exceeds expectation")); + return Err(array::value_err("bitset size exceeds expectation".to_string())); } let end: u64 = std::cmp::min(begin + 64, self.nr_entries); @@ -44,7 +43,7 @@ pub fn read_bitset( root: u64, ignore_none_fatal: bool, bitset: &mut FixedBitSet, -)-> btree::Result<()> { +)-> array::Result<()> { let w = ArrayWalker::new(engine.clone(), ignore_none_fatal); let mut v = BitsetVisitor::new(bitset); w.walk(&mut v, root) diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs index 9d13c33..ef04f6b 100644 --- a/src/pdata/mod.rs +++ b/src/pdata/mod.rs @@ -1,4 +1,4 @@ -pub mod array_block; +pub mod array; pub mod array_walker; pub mod bitset_walker; pub mod btree;