emo
/
thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[cache_check (rust)] Detect structural errors on arrays 

- Define error types for arrays
- Propagate low-level errors to tools
- Rename array_block.rs to array.rs
This commit is contained in:
Ming-Hung Tsai 2021-04-13 13:39:14 +08:00
parent ace9c1d1e3
commit 9b4a0607ea
10 changed files with 249 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
src/cache/check.rs vendored
View File

@ -8,6 +8,7 @@ use crate::cache::hint::*;
use crate::cache::mapping::*;
use crate::cache::superblock::*;
use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
use crate::pdata::array;
use crate::pdata::array_walker::*;
use crate::pdata::bitset_walker::*;
@ -33,30 +34,30 @@ mod format1 {
}
}
fn check_flags(&self, m: &Mapping) -> anyhow::Result<()> {
fn check_flags(&self, m: &Mapping) -> array::Result<()> {
if (m.flags & !(MappingFlags::Valid as u32 | MappingFlags::Dirty as u32)) != 0 {
return Err(anyhow!("unknown flags in mapping: {}", m.flags));
return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags)));
}
if !m.is_valid() && m.is_dirty() {
return Err(anyhow!("dirty bit found on an unmapped block"));
return Err(array::value_err(format!("dirty bit found on an unmapped block")));
}
Ok(())
}
fn check_oblock(&self, m: &Mapping) -> anyhow::Result<()> {
fn check_oblock(&self, m: &Mapping) -> array::Result<()> {
if !m.is_valid() {
if m.oblock > 0 {
return Err(anyhow!("invalid mapped block"));
return Err(array::value_err(format!("invalid block is mapped")));
}
return Ok(());
}
if m.oblock >= self.nr_origin_blocks {
return Err(anyhow!("mapping beyond end of the origin device"));
return Err(array::value_err(format!("mapping beyond end of the origin device")));
}
let mut seen_oblocks = self.seen_oblocks.lock().unwrap();
if seen_oblocks.contains(&m.oblock) {
return Err(anyhow!("origin block already mapped"));
return Err(array::value_err(format!("origin block already mapped")));
}
seen_oblocks.insert(m.oblock);
@ -65,7 +66,7 @@ mod format1 {
}
impl ArrayVisitor<Mapping> for MappingChecker {
fn visit(&self, _index: u64, m: Mapping) -> anyhow::Result<()> {
fn visit(&self, _index: u64, m: Mapping) -> array::Result<()> {
self.check_flags(&m)?;
self.check_oblock(&m)?;
@ -98,28 +99,28 @@ mod format2 {
}
}
fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> anyhow::Result<()> {
fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> array::Result<()> {
if (m.flags & !(MappingFlags::Valid as u32)) != 0 {
return Err(anyhow!("unknown flags in mapping: {}", m.flags));
return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags)));
}
if !m.is_valid() && dirty_bit {
return Err(anyhow!("dirty bit found on an unmapped block"));
return Err(array::value_err(format!("dirty bit found on an unmapped block")));
}
Ok(())
}
fn check_oblock(&self, m: &Mapping, seen_oblocks: &mut BTreeSet<u64>) -> anyhow::Result<()> {
fn check_oblock(&self, m: &Mapping, seen_oblocks: &mut BTreeSet<u64>) -> array::Result<()> {
if !m.is_valid() {
if m.oblock > 0 {
return Err(anyhow!("invalid mapped block"));
return Err(array::value_err(format!("invalid mapped block")));
}
return Ok(());
}
if m.oblock >= self.nr_origin_blocks {
return Err(anyhow!("mapping beyond end of the origin device"));
return Err(array::value_err(format!("mapping beyond end of the origin device")));
}
if seen_oblocks.contains(&m.oblock) {
return Err(anyhow!("origin block already mapped"));
return Err(array::value_err(format!("origin block already mapped")));
}
seen_oblocks.insert(m.oblock);
@ -128,7 +129,7 @@ mod format2 {
}
impl ArrayVisitor<Mapping> for MappingChecker {
fn visit(&self, index: u64, m: Mapping) -> anyhow::Result<()> {
fn visit(&self, index: u64, m: Mapping) -> array::Result<()> {
let mut inner = self.inner.lock().unwrap();
self.check_flags(&m, inner.dirty_bits.contains(index as usize))?;
self.check_oblock(&m, &mut inner.seen_oblocks)?;
@ -149,7 +150,7 @@ impl HintChecker {
}
impl ArrayVisitor<Hint> for HintChecker {
fn visit(&self, _index: u64, _hint: Hint) -> anyhow::Result<()> {
fn visit(&self, _index: u64, _hint: Hint) -> array::Result<()> {
// TODO: check hints
Ok(())
}

21
src/cache/dump.rs vendored
View File

@ -1,4 +1,4 @@
use anyhow::{anyhow, Result};
use anyhow::anyhow;
use fixedbitset::FixedBitSet;
use std::path::Path;
use std::sync::{Arc, Mutex};
@ -8,6 +8,7 @@ use crate::cache::mapping::Mapping;
use crate::cache::superblock::*;
use crate::cache::xml::{self, MetadataVisitor};
use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
use crate::pdata::array;
use crate::pdata::array_walker::*;
//------------------------------------------
@ -45,7 +46,7 @@ mod format1 {
}
impl<'a> ArrayVisitor<Mapping> for MappingEmitter<'a> {
fn visit(&self, index: u64, m: Mapping) -> Result<()> {
fn visit(&self, index: u64, m: Mapping) -> array::Result<()> {
if m.is_valid() {
let m = xml::Map {
cblock: index as u32,
@ -55,7 +56,7 @@ mod format1 {
let mut inner = self.inner.lock().unwrap();
inner.valid_mappings.set(index as usize, true);
inner.visitor.mapping(&m)?;
inner.visitor.mapping(&m).map_err(|e| array::value_err(format!("{}", e)))?;
}
Ok(())
@ -89,7 +90,7 @@ mod format2 {
}
impl ArrayVisitor<u64> for DirtyVisitor {
fn visit(&self, index: u64, bits: u64) -> Result<()> {
fn visit(&self, index: u64, bits: u64) -> array::Result<()> {
for i in 0..64u64 {
if (index + i) >= self.nr_entries as u64 {
break;
@ -132,7 +133,7 @@ mod format2 {
}
impl<'a> ArrayVisitor<Mapping> for MappingEmitter<'a> {
fn visit(&self, index: u64, m: Mapping) -> Result<()> {
fn visit(&self, index: u64, m: Mapping) -> array::Result<()> {
if m.is_valid() {
let mut inner = self.inner.lock().unwrap();
let dirty = inner.dirty_bits.contains(index as usize);
@ -143,7 +144,7 @@ mod format2 {
};
inner.valid_mappings.set(index as usize, true);
inner.visitor.mapping(&m)?;
inner.visitor.mapping(&m).map_err(|e| array::value_err(format!("{}", e)))?;
}
Ok(())
@ -168,14 +169,18 @@ impl<'a> HintEmitter<'a> {
}
impl<'a> ArrayVisitor<Hint> for HintEmitter<'a> {
fn visit(&self, index: u64, hint: Hint) -> anyhow::Result<()> {
fn visit(&self, index: u64, hint: Hint) -> array::Result<()> {
if self.valid_mappings.contains(index as usize) {
let h = xml::Hint {
cblock: index as u32,
data: hint.hint.to_vec(),
};
self.emitter.lock().unwrap().hint(&h)?;
self.emitter
.lock()
.unwrap()
.hint(&h)
.map_err(|e| array::value_err(format!("{}", e)))?;
}
Ok(())

4
src/checksum.rs
View File

@ -11,6 +11,7 @@ const SUPERBLOCK_CSUM_XOR: u32 = 160774;
const BITMAP_CSUM_XOR: u32 = 240779;
const INDEX_CSUM_XOR: u32 = 160478;
const BTREE_CSUM_XOR: u32 = 121107;
const ARRAY_CSUM_XOR: u32 = 595846735;
fn checksum(buf: &[u8]) -> u32 {
crc32c(&buf[4..]) ^ 0xffffffff
@ -22,6 +23,7 @@ pub enum BT {
NODE,
INDEX,
BITMAP,
ARRAY,
UNKNOWN,
}
@ -41,6 +43,7 @@ pub fn metadata_block_type(buf: &[u8]) -> BT {
BTREE_CSUM_XOR => BT::NODE,
BITMAP_CSUM_XOR => BT::BITMAP,
INDEX_CSUM_XOR => BT::INDEX,
ARRAY_CSUM_XOR => BT::ARRAY,
_ => BT::UNKNOWN,
}
}
@ -56,6 +59,7 @@ pub fn write_checksum(buf: &mut [u8], kind: BT) -> Result<()> {
NODE => BTREE_CSUM_XOR,
BITMAP => BITMAP_CSUM_XOR,
INDEX => INDEX_CSUM_XOR,
ARRAY => ARRAY_CSUM_XOR,
UNKNOWN => {return Err(anyhow!("Invalid block type"));}
};

4
src/pack/node_encode.rs
View File

@ -114,4 +114,8 @@ pub fn pack_index<W: Write>(w: &mut W, bytes: &[u8]) -> PResult<()> {
io_to_pr(pack_literal(w, bytes))
}
pub fn pack_array<W: Write>(w: &mut W, bytes: &[u8]) -> PResult<()> {
io_to_pr(pack_literal(w, bytes))
}
//-------------------------------------

1
src/pack/toplevel.rs
View File

@ -209,6 +209,7 @@ fn pack_block<W: Write>(w: &mut W, kind: BT, buf: &[u8]) -> Result<()> {
BT::NODE => pack_btree_node(w, buf).context("unable to pack btree node")?,
BT::INDEX => pack_index(w, buf).context("unable to pack space map index")?,
BT::BITMAP => pack_bitmap(w, buf).context("unable to pack space map bitmap")?,
BT::ARRAY => pack_array(w, buf).context("unable to pack array block")?,
BT::UNKNOWN => return Err(anyhow!("asked to pack an unknown block type")),
}

133
src/pdata/array.rs Normal file
View File

@ -0,0 +1,133 @@
use nom::{multi::count, number::complete::*, IResult};
use thiserror::Error;
use crate::io_engine::BLOCK_SIZE;
use crate::pdata::btree;
use crate::pdata::unpack::Unpack;
//------------------------------------------
const ARRAY_BLOCK_HEADER_SIZE: u32 = 24;
pub struct ArrayBlockHeader {
pub csum: u32,
pub max_entries: u32,
pub nr_entries: u32,
pub value_size: u32,
pub blocknr: u64,
}
impl Unpack for ArrayBlockHeader {
fn disk_size() -> u32 {
ARRAY_BLOCK_HEADER_SIZE
}
fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> {
let (i, csum) = le_u32(data)?;
let (i, max_entries) = le_u32(i)?;
let (i, nr_entries) = le_u32(i)?;
let (i, value_size) = le_u32(i)?;
let (i, blocknr) = le_u64(i)?;
Ok((
i,
ArrayBlockHeader {
csum,
max_entries,
nr_entries,
value_size,
blocknr,
},
))
}
}
pub struct ArrayBlock<V: Unpack> {
pub header: ArrayBlockHeader,
pub values: Vec<V>,
}
//------------------------------------------
#[derive(Error, Clone, Debug)]
pub enum ArrayError {
#[error("io_error")]
IoError,
#[error("block error: {0}")]
BlockError(String),
#[error("value error: {0}")]
ValueError(String),
#[error("aggregate: {0:?}")]
Aggregate(Vec<ArrayError>),
#[error("{0:?}, {1}")]
Path(Vec<u64>, Box<ArrayError>),
#[error(transparent)]
BTreeError(#[from] btree::BTreeError),
}
pub fn array_block_err(path: &[u64], msg: &str) -> ArrayError {
ArrayError::Path(
path.to_vec(),
Box::new(ArrayError::BlockError(msg.to_string())),
)
}
pub fn value_err(msg: String) -> ArrayError {
ArrayError::ValueError(msg)
}
pub fn aggregate_error(errs: Vec<ArrayError>) -> ArrayError {
ArrayError::Aggregate(errs)
}
pub type Result<T> = std::result::Result<T, ArrayError>;
//------------------------------------------
fn convert_result<'a, V>(path: &[u64], r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
r.map_err(|_| array_block_err(path, "parse error"))
}
pub fn unpack_array_block<V: Unpack>(path: &[u64], data: &[u8]) -> Result<ArrayBlock<V>> {
// TODO: collect errors
let (i, header) =
ArrayBlockHeader::unpack(data).map_err(|_| array_block_err(
path,
"Couldn't parse array block header"
))?;
// check value_size
if header.value_size != V::disk_size() {
return Err(array_block_err(
path,
&format!(
"value_size mismatch: expected {}, was {}",
V::disk_size(),
header.value_size
)
));
}
// check max_entries
if header.value_size * header.max_entries + ARRAY_BLOCK_HEADER_SIZE > BLOCK_SIZE as u32 {
return Err(array_block_err(
path,
&format!("max_entries is too large ({})", header.max_entries)
));
}
// TODO: check nr_entries < max_entries
// TODO: check block_nr
let (_i, values) = convert_result(path, count(V::unpack, header.nr_entries as usize)(i))?;
Ok(ArrayBlock { header, values })
}
//------------------------------------------

60
src/pdata/array_block.rs
View File

@ -1,60 +0,0 @@
use anyhow::anyhow;
use anyhow::Result;
use nom::{multi::count, number::complete::*, IResult};
use crate::pdata::unpack::Unpack;
//------------------------------------------
pub struct ArrayBlockHeader {
pub csum: u32,
pub max_entries: u32,
pub nr_entries: u32,
pub value_size: u32,
pub blocknr: u64,
}
impl Unpack for ArrayBlockHeader {
fn disk_size() -> u32 {
24
}
fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> {
let (i, csum) = le_u32(data)?;
let (i, max_entries) = le_u32(i)?;
let (i, nr_entries) = le_u32(i)?;
let (i, value_size) = le_u32(i)?;
let (i, blocknr) = le_u64(i)?;
Ok((
i,
ArrayBlockHeader {
csum,
max_entries,
nr_entries,
value_size,
blocknr,
},
))
}
}
pub struct ArrayBlock<V: Unpack> {
pub header: ArrayBlockHeader,
pub values: Vec<V>,
}
fn convert_result<V>(r: IResult<&[u8], V>) -> Result<(&[u8], V)> {
r.map_err(|_| anyhow!("parse error"))
}
pub fn unpack_array_block<V: Unpack>(data: &[u8]) -> Result<ArrayBlock<V>> {
// TODO: collect errors
let (i, header) =
ArrayBlockHeader::unpack(data).map_err(|_e| anyhow!("Couldn't parse header"))?;
let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?;
Ok(ArrayBlock { header, values })
}
//------------------------------------------

92
src/pdata/array_walker.rs
View File

@ -1,8 +1,9 @@
use std::sync::Arc;
use crate::checksum;
use crate::io_engine::*;
use crate::pdata::array_block::*;
use crate::pdata::btree::*;
use crate::pdata::array::{self, *};
use crate::pdata::btree::{self, *};
use crate::pdata::btree_walker::*;
use crate::pdata::unpack::*;
@ -15,12 +16,12 @@ pub struct ArrayWalker {
// FIXME: define another Result type for array visiting?
pub trait ArrayVisitor<V: Unpack> {
fn visit(&self, index: u64, v: V) -> anyhow::Result<()>;
fn visit(&self, index: u64, v: V) -> array::Result<()>;
}
struct BlockValueVisitor<'a, V> {
engine: Arc<dyn IoEngine + Send + Sync>,
array_block_visitor: &'a mut dyn ArrayVisitor<V>,
array_visitor: &'a mut dyn ArrayVisitor<V>,
}
impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> {
@ -30,22 +31,37 @@ impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> {
) -> BlockValueVisitor<'a, V> {
BlockValueVisitor {
engine: e,
array_block_visitor: v,
array_visitor: v,
}
}
pub fn visit_array_block(&self, index: u64, array_block: ArrayBlock<V>) {
pub fn visit_array_block(&self, index: u64, array_block: ArrayBlock<V>) -> array::Result<()>{
let mut errs: Vec<ArrayError> = Vec::new();
let begin = index * array_block.header.max_entries as u64;
for i in 0..array_block.header.nr_entries {
self.array_block_visitor
.visit(begin + i as u64, array_block.values[i as usize])
.unwrap();
if let Err(e) = self.array_visitor.visit(begin + i as u64, array_block.values[i as usize]) {
errs.push(e); // TODO: add path or keys context?
}
}
// FIXME: duplicate to BTreeWalker::build_aggregrate()
match errs.len() {
0 => Ok(()),
1 => {
let e = errs[0].clone();
Err(e)
}
_ => {
let e = array::aggregate_error(errs);
Err(e)
}
}
}
}
impl<'a, V: Unpack + Copy> NodeVisitor<u64> for BlockValueVisitor<'a, V> {
// FIXME: return errors
// FIXME: wrap ArrayError into BTreeError, rather than mapping to value_err?
fn visit(
&self,
path: &[u64],
@ -53,20 +69,54 @@ impl<'a, V: Unpack + Copy> NodeVisitor<u64> for BlockValueVisitor<'a, V> {
_h: &NodeHeader,
keys: &[u64],
values: &[u64],
) -> Result<()> {
) -> btree::Result<()> {
let mut path = path.to_vec();
let mut errs: Vec<BTreeError> = Vec::new();
for (n, index) in keys.iter().enumerate() {
let b = self.engine.read(values[n]).map_err(|_| io_err(path))?;
let array_block = unpack_array_block::<V>(b.get_data()).map_err(|_| io_err(path))?;
self.visit_array_block(*index, array_block);
let b = self.engine.read(values[n]).map_err(|_| io_err(&path))?;
// FIXME: move to unpack_array_block?
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::ARRAY {
errs.push(btree::value_err(
format!("checksum failed for array block {}, {:?}", b.loc, bt)
));
}
path.push(values[n]);
match unpack_array_block::<V>(&path, b.get_data()) {
Ok(array_block) => {
if let Err(e) = self.visit_array_block(*index, array_block) {
errs.push(btree::value_err(format!("{}", e)));
}
},
Err(e) => {
errs.push(btree::value_err(format!("{}", e)));
}
}
path.pop();
}
// FIXME: duplicate to BTreeWalker::build_aggregrate()
match errs.len() {
0 => Ok(()),
1 => {
let e = errs[0].clone();
Err(e)
}
_ => {
let e = btree::aggregate_error(errs);
Err(e)
}
}
}
fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> {
Ok(())
}
fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> {
Ok(())
}
fn end_walk(&self) -> Result<()> {
fn end_walk(&self) -> btree::Result<()> {
Ok(())
}
}
@ -81,7 +131,7 @@ impl ArrayWalker {
}
// FIXME: redefine the Result type for array visiting?
pub fn walk<V>(&self, visitor: &mut dyn ArrayVisitor<V>, root: u64) -> Result<()>
pub fn walk<V>(&self, visitor: &mut dyn ArrayVisitor<V>, root: u64) -> array::Result<()>
where
V: Unpack + Copy,
{
@ -89,7 +139,7 @@ impl ArrayWalker {
let mut path = Vec::new();
path.push(0);
let v = BlockValueVisitor::<V>::new(self.engine.clone(), visitor);
w.walk(&mut path, &v, root)
w.walk(&mut path, &v, root).map_err(|e| ArrayError::BTreeError(e))
}
}

9
src/pdata/bitset_walker.rs
View File

@ -1,10 +1,9 @@
use anyhow::{anyhow, Result};
use fixedbitset::FixedBitSet;
use std::sync::{Arc, Mutex};
use crate::io_engine::IoEngine;
use crate::pdata::array;
use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker};
use crate::pdata::btree::{self};
struct BitsetVisitor<'a> {
nr_entries: u64,
@ -21,10 +20,10 @@ impl<'a> BitsetVisitor<'a> {
}
impl<'a> ArrayVisitor<u64> for BitsetVisitor<'a> {
fn visit(&self, index: u64, bits: u64) -> Result<()> {
fn visit(&self, index: u64, bits: u64) -> array::Result<()> {
let begin: u64 = index << 6;
if begin > self.nr_entries {
return Err(anyhow!("bitset size exceeds expectation"));
return Err(array::value_err("bitset size exceeds expectation".to_string()));
}
let end: u64 = std::cmp::min(begin + 64, self.nr_entries);
@ -44,7 +43,7 @@ pub fn read_bitset(
root: u64,
ignore_none_fatal: bool,
bitset: &mut FixedBitSet,
)-> btree::Result<()> {
)-> array::Result<()> {
let w = ArrayWalker::new(engine.clone(), ignore_none_fatal);
let mut v = BitsetVisitor::new(bitset);
w.walk(&mut v, root)

2
src/pdata/mod.rs
View File

@ -1,4 +1,4 @@
pub mod array_block;
pub mod array;
pub mod array_walker;
pub mod bitset_walker;
pub mod btree;