From 239ff7dfa172b04b8d2bf629ca2dc67b4d4ff1ac Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 20 Apr 2021 16:07:25 +0800 Subject: [PATCH] [cache_check (rust)] Add more checks - Report errors - Support reading partially broken bitset - The output is a bitmap of 2-bit entries, indicating availability of bits --- src/bin/cache_check.rs | 20 ++++++++++++++ src/cache/check.rs | 54 +++++++++++++++++++++++++++----------- src/pdata/array.rs | 29 +++++++++++++++++---- src/pdata/bitset.rs | 59 ++++++++++++++++++++++++++++++++---------- 4 files changed, 129 insertions(+), 33 deletions(-) diff --git a/src/bin/cache_check.rs b/src/bin/cache_check.rs index 4fed20c..f142f39 100644 --- a/src/bin/cache_check.rs +++ b/src/bin/cache_check.rs @@ -1,8 +1,12 @@ extern crate clap; extern crate thinp; +use atty::Stream; use clap::{App, Arg}; use std::path::Path; +use std::sync::Arc; + +use thinp::report::*; use thinp::cache::check::{check, CacheCheckOptions}; //------------------------------------------ @@ -39,11 +43,26 @@ fn main() { .help("Don't check the discard bitset") .long("skip-discards") .value_name("SKIP_DISCARDS"), + ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), ); let matches = parser.get_matches(); let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let report; + if matches.is_present("QUIET") { + report = std::sync::Arc::new(mk_quiet_report()); + } else if atty::is(Stream::Stdout) { + report = std::sync::Arc::new(mk_progress_bar_report()); + } else { + report = Arc::new(mk_simple_report()); + } + let opts = CacheCheckOptions { dev: &input_file, async_io: false, @@ -51,6 +70,7 @@ fn main() { skip_mappings: matches.is_present("SKIP_MAPPINGS"), skip_hints: matches.is_present("SKIP_HINTS"), skip_discards: matches.is_present("SKIP_DISCARDS"), + report, }; if let Err(reason) = check(opts) { diff --git a/src/cache/check.rs b/src/cache/check.rs index 01df923..b91504f 100644 --- a/src/cache/check.rs +++ b/src/cache/check.rs @@ -1,5 +1,4 @@ use anyhow::anyhow; -use fixedbitset::FixedBitSet; use std::collections::BTreeSet; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -11,6 +10,7 @@ use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock, ArrayError}; use crate::pdata::array_walker::*; use crate::pdata::bitset::*; +use crate::report::*; //------------------------------------------ @@ -102,11 +102,11 @@ mod format2 { struct Inner { seen_oblocks: BTreeSet, - dirty_bits: FixedBitSet, + dirty_bits: CheckedBitSet, } impl MappingChecker { - pub fn new(nr_origin_blocks: Option, dirty_bits: FixedBitSet) -> MappingChecker { + pub fn new(nr_origin_blocks: Option, dirty_bits: CheckedBitSet) -> MappingChecker { MappingChecker { nr_origin_blocks: if let Some(n) = nr_origin_blocks {n} else {MAX_ORIGIN_BLOCKS}, inner: Mutex::new(Inner { @@ -116,11 +116,11 @@ mod format2 { } } - fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> array::Result<()> { + fn check_flags(&self, m: &Mapping, dirty_bit: Option) -> array::Result<()> { if (m.flags & !(MappingFlags::Valid as u32)) != 0 { return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags))); } - if !m.is_valid() && dirty_bit { + if !m.is_valid() && dirty_bit.is_some() && dirty_bit.unwrap() { return Err(array::value_err(format!("dirty bit found on an unmapped block"))); } Ok(()) @@ -203,10 +203,12 @@ pub struct CacheCheckOptions<'a> { pub skip_mappings: bool, pub skip_hints: bool, pub skip_discards: bool, + pub report: Arc, } -// TODO: thread pool, report +// TODO: thread pool struct Context { + report: Arc, engine: Arc, } @@ -220,7 +222,10 @@ fn mk_context(opts: &CacheCheckOptions) -> anyhow::Result { engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads, false)?); } - Ok(Context { engine }) + Ok(Context { + report: opts.report.clone(), + engine, + }) } fn check_superblock(sb: &Superblock) -> anyhow::Result<()> { @@ -256,15 +261,25 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { match sb.version { 1 => { let mut c = format1::MappingChecker::new(nr_origin_blocks); - w.walk(&mut c, sb.mapping_root)?; + if let Err(e) = w.walk(&mut c, sb.mapping_root) { + ctx.report.fatal(&format!("{}", e)); + } } 2 => { - // FIXME: possibly size truncation on 32-bit targets - let mut dirty_bits = FixedBitSet::with_capacity(sb.cache_blocks as usize); // TODO: allow ignore_none_fatal - read_bitset(engine.clone(), sb.dirty_root.unwrap(), false, &mut dirty_bits)?; + let (dirty_bits, err) = read_bitset( + engine.clone(), + sb.dirty_root.unwrap(), + sb.cache_blocks as usize, + false, + ); + if err.is_some() { + ctx.report.fatal(&format!("{}", err.unwrap())); + } let mut c = format2::MappingChecker::new(nr_origin_blocks, dirty_bits); - w.walk(&mut c, sb.mapping_root)?; + if let Err(e) = w.walk(&mut c, sb.mapping_root) { + ctx.report.fatal(&format!("{}", e)); + } } v => { return Err(anyhow!("unsupported metadata version {}", v)); @@ -278,14 +293,23 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { } let w = ArrayWalker::new(engine.clone(), false); let mut c = HintChecker::new(); - w.walk(&mut c, sb.hint_root)?; + if let Err(e) = w.walk(&mut c, sb.hint_root) { + ctx.report.fatal(&format!("{}", e)); + } } // The discard bitset might not be available if the cache has never been suspended, // e.g., a crash of freshly created cache. if !opts.skip_discards && sb.discard_root != 0 { - let mut discard_bits = FixedBitSet::with_capacity(sb.discard_nr_blocks as usize); - read_bitset(engine.clone(), sb.discard_root, false, &mut discard_bits)?; + let (discard_bits, err) = read_bitset( + engine.clone(), + sb.discard_root, + sb.cache_blocks as usize, + false, + ); + if err.is_some() { + ctx.report.fatal(&format!("{}", err.unwrap())); + } } Ok(()) diff --git a/src/pdata/array.rs b/src/pdata/array.rs index 4a977f6..788d239 100644 --- a/src/pdata/array.rs +++ b/src/pdata/array.rs @@ -1,4 +1,5 @@ use nom::{multi::count, number::complete::*, IResult}; +use std::fmt; use thiserror::Error; use crate::checksum; @@ -52,25 +53,43 @@ pub struct ArrayBlock { #[derive(Error, Clone, Debug)] pub enum ArrayError { - #[error("io_error")] + //#[error("io_error")] IoError, - #[error("block error: {0}")] + //#[error("block error: {0}")] BlockError(String), - #[error("value error: {0}")] + //#[error("value error: {0}")] ValueError(String), - #[error("aggregate: {0:?}")] + //#[error("aggregate: {0:?}")] Aggregate(Vec), - #[error("{0:?}, {1}")] + //#[error("{0:?}, {1}")] Path(Vec, Box), #[error(transparent)] BTreeError(#[from] btree::BTreeError), } +impl fmt::Display for ArrayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ArrayError::IoError => write!(f, "io error"), + ArrayError::BlockError(msg) => write!(f, "block error: {}", msg), + ArrayError::ValueError(msg) => write!(f, "value error: {}", msg), + ArrayError::Aggregate(errs) => { + for e in errs { + write!(f, "{}", e)? + } + Ok(()) + } + ArrayError::Path(path, e) => write!(f, "{} {}", e, btree::encode_node_path(path)), + ArrayError::BTreeError(e) => write!(f, "{}", e), + } + } +} + pub fn array_block_err(path: &[u64], msg: &str) -> ArrayError { ArrayError::Path( path.to_vec(), diff --git a/src/pdata/bitset.rs b/src/pdata/bitset.rs index 7cc7b7f..013d9ec 100644 --- a/src/pdata/bitset.rs +++ b/src/pdata/bitset.rs @@ -5,21 +5,49 @@ use crate::io_engine::IoEngine; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker}; -struct BitsetVisitor<'a> { - nr_entries: u64, - bits: Mutex<&'a mut FixedBitSet>, +pub struct CheckedBitSet { + bits: FixedBitSet, } -impl<'a> BitsetVisitor<'a> { - pub fn new(bitset: &'a mut FixedBitSet) -> Self { - BitsetVisitor { - nr_entries: bitset.len() as u64, - bits: Mutex::new(bitset), +impl CheckedBitSet { + pub fn with_capacity(bits: usize) -> CheckedBitSet { + CheckedBitSet { + bits: FixedBitSet::with_capacity(bits << 1), } } + + pub fn set(&mut self, bit: usize, enabled: bool) { + self.bits.set(bit << 1, true); + self.bits.set((bit << 1) + 1, enabled); + } + + pub fn contains(&self, bit: usize) -> Option { + if !self.bits.contains(bit << 1) { + return None; + } + Some(self.bits.contains((bit << 1) + 1)) + } } -impl<'a> ArrayVisitor for BitsetVisitor<'a> { +struct BitsetVisitor { + nr_entries: usize, + bits: Mutex, +} + +impl BitsetVisitor { + pub fn new(nr_entries: usize) -> Self { + BitsetVisitor { + nr_entries, + bits: Mutex::new(CheckedBitSet::with_capacity(nr_entries)), + } + } + + pub fn get_bitset(self) -> CheckedBitSet { + self.bits.into_inner().unwrap() + } +} + +impl ArrayVisitor for BitsetVisitor { fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { let mut begin = index as usize * (b.header.max_entries as usize) << 6; @@ -46,10 +74,15 @@ impl<'a> ArrayVisitor for BitsetVisitor<'a> { pub fn read_bitset( engine: Arc, root: u64, + nr_entries: usize, ignore_none_fatal: bool, - bitset: &mut FixedBitSet, -)-> array::Result<()> { +)-> (CheckedBitSet, Option) { let w = ArrayWalker::new(engine.clone(), ignore_none_fatal); - let mut v = BitsetVisitor::new(bitset); - w.walk(&mut v, root) + let mut v = BitsetVisitor::new(nr_entries); + let err = w.walk(&mut v, root); + let e = match err { + Ok(()) => None, + Err(e) => Some(e), + }; + return (v.get_bitset(), e); }