From cf4b937ade63a9dd1e7a0d9e46347f5696ce5761 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 21 Apr 2021 23:36:10 +0800 Subject: [PATCH] [cache_check (rust)] Check space map counts - Support space map checking and auto-repair --- src/bin/cache_check.rs | 6 ++++++ src/cache/check.rs | 45 +++++++++++++++++++++++++++++++++------ src/pdata/array_walker.rs | 34 +++++++++++++++++++++++++++-- src/pdata/bitset.rs | 21 +++++++++++++++++- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/bin/cache_check.rs b/src/bin/cache_check.rs index 452a495..2c70358 100644 --- a/src/bin/cache_check.rs +++ b/src/bin/cache_check.rs @@ -49,6 +49,11 @@ fn main() { .help("Only return a non-zero exit code if a fatal error is found.") .long("ignore-non-fatal-errors"), ) + .arg( + Arg::with_name("AUTO_REPAIR") + .help("Auto repair trivial issues.") + .long("auto-repair"), + ) .arg( Arg::with_name("QUIET") .help("Suppress output messages, return only exit code.") @@ -76,6 +81,7 @@ fn main() { skip_hints: matches.is_present("SKIP_HINTS"), skip_discards: matches.is_present("SKIP_DISCARDS"), ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"), + auto_repair: matches.is_present("AUTO_REPAIR"), report, }; diff --git a/src/cache/check.rs b/src/cache/check.rs index 71a59ae..2fce400 100644 --- a/src/cache/check.rs +++ b/src/cache/check.rs @@ -10,6 +10,9 @@ use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock, ArrayError}; use crate::pdata::array_walker::*; use crate::pdata::bitset::*; +use crate::pdata::space_map::*; +use crate::pdata::space_map_checker::*; +use crate::pdata::unpack::unpack; use crate::report::*; //------------------------------------------ @@ -18,6 +21,14 @@ const MAX_CONCURRENT_IO: u32 = 1024; //------------------------------------------ +fn inc_superblock(sm: &ASpaceMap) -> anyhow::Result<()> { + let mut sm = sm.lock().unwrap(); + sm.inc(SUPERBLOCK_LOCATION, 1)?; + Ok(()) +} + +//------------------------------------------ + mod format1 { use super::*; @@ -204,6 +215,7 @@ pub struct CacheCheckOptions<'a> { pub skip_hints: bool, pub skip_discards: bool, pub ignore_non_fatal: bool, + pub auto_repair: bool, pub report: Arc, } @@ -240,6 +252,8 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { let ctx = mk_context(&opts)?; let engine = &ctx.engine; + let metadata_sm = core_sm(engine.get_nr_blocks(), u8::MAX as u32); + inc_superblock(&metadata_sm)?; let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; check_superblock(&sb)?; @@ -258,7 +272,7 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { // TODO: factor out into check_mappings() if !opts.skip_mappings { - let w = ArrayWalker::new(engine.clone(), opts.ignore_non_fatal); + let w = ArrayWalker::new_with_sm(engine.clone(), metadata_sm.clone(), opts.ignore_non_fatal)?; match sb.version { 1 => { let mut c = format1::MappingChecker::new(nr_origin_blocks); @@ -267,13 +281,13 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { } } 2 => { - // TODO: allow ignore_none_fatal - let (dirty_bits, err) = read_bitset( + let (dirty_bits, err) = read_bitset_with_sm( engine.clone(), sb.dirty_root.unwrap(), sb.cache_blocks as usize, + metadata_sm.clone(), opts.ignore_non_fatal, - ); + )?; if err.is_some() { ctx.report.fatal(&format!("{}", err.unwrap())); } @@ -292,7 +306,7 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { if sb.policy_hint_size != 4 { return Err(anyhow!("cache_check only supports policy hint size of 4")); } - let w = ArrayWalker::new(engine.clone(), opts.ignore_non_fatal); + let w = ArrayWalker::new_with_sm(engine.clone(), metadata_sm.clone(), opts.ignore_non_fatal)?; let mut c = HintChecker::new(); if let Err(e) = w.walk(&mut c, sb.hint_root) { ctx.report.fatal(&format!("{}", e)); @@ -302,17 +316,34 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { // The discard bitset might not be available if the cache has never been suspended, // e.g., a crash of freshly created cache. if !opts.skip_discards && sb.discard_root != 0 { - let (discard_bits, err) = read_bitset( + let (_discard_bits, err) = read_bitset_with_sm( engine.clone(), sb.discard_root, sb.cache_blocks as usize, + metadata_sm.clone(), opts.ignore_non_fatal, - ); + )?; if err.is_some() { ctx.report.fatal(&format!("{}", err.unwrap())); } } + let root = unpack::(&sb.metadata_sm_root[0..])?; + let metadata_leaks = check_metadata_space_map( + engine.clone(), + ctx.report.clone(), + root, + metadata_sm.clone(), + opts.ignore_non_fatal, + )?; + + if opts.auto_repair { + if !metadata_leaks.is_empty() { + ctx.report.info("Repairing metadata leaks."); + repair_space_map(ctx.engine.clone(), metadata_leaks, metadata_sm.clone())?; + } + } + Ok(()) } diff --git a/src/pdata/array_walker.rs b/src/pdata/array_walker.rs index 34df482..f8d029f 100644 --- a/src/pdata/array_walker.rs +++ b/src/pdata/array_walker.rs @@ -4,12 +4,14 @@ use crate::io_engine::*; use crate::pdata::array::{self, *}; use crate::pdata::btree::{self, *}; use crate::pdata::btree_walker::*; +use crate::pdata::space_map::*; use crate::pdata::unpack::*; //------------------------------------------ pub struct ArrayWalker { engine: Arc, + sm: Arc>, ignore_non_fatal: bool, } @@ -23,17 +25,20 @@ pub trait ArrayVisitor { struct BlockValueVisitor<'a, V> { engine: Arc, array_visitor: &'a mut dyn ArrayVisitor, + sm: Arc>, array_errs: Mutex>, } impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> { pub fn new( e: Arc, + sm: Arc>, v: &'a mut dyn ArrayVisitor, ) -> BlockValueVisitor<'a, V> { BlockValueVisitor { engine: e, array_visitor: v, + sm: sm, array_errs: Mutex::new(Vec::new()), } } @@ -85,6 +90,8 @@ impl<'a, V: Unpack + Copy> NodeVisitor for BlockValueVisitor<'a, V> { if let Err(e) = self.array_visitor.visit(keys[i], array_block) { self.array_errs.lock().unwrap().push(e); } + let mut sm = self.sm.lock().unwrap(); + sm.inc(b.loc, 1).unwrap(); }, Err(e) => { self.array_errs.lock().unwrap().push(e); @@ -113,21 +120,44 @@ impl<'a, V: Unpack + Copy> NodeVisitor for BlockValueVisitor<'a, V> { impl ArrayWalker { pub fn new(engine: Arc, ignore_non_fatal: bool) -> ArrayWalker { + let nr_blocks = engine.get_nr_blocks() as u64; let r: ArrayWalker = ArrayWalker { engine, + sm: Arc::new(Mutex::new(RestrictedSpaceMap::new(nr_blocks))), ignore_non_fatal, }; r } + pub fn new_with_sm( + engine: Arc, + sm: Arc>, + ignore_non_fatal: bool, + ) -> array::Result { + { + let sm = sm.lock().unwrap(); + assert_eq!(sm.get_nr_blocks().unwrap(), engine.get_nr_blocks()); + } + + Ok(ArrayWalker { + engine, + sm, + ignore_non_fatal, + }) + } + pub fn walk(&self, visitor: &mut dyn ArrayVisitor, root: u64) -> array::Result<()> where V: Unpack + Copy, { - let w = BTreeWalker::new(self.engine.clone(), self.ignore_non_fatal); + let w = BTreeWalker::new_with_sm( + self.engine.clone(), + self.sm.clone(), + self.ignore_non_fatal + )?; let mut path = Vec::new(); path.push(0); - let v = BlockValueVisitor::::new(self.engine.clone(), visitor); + let v = BlockValueVisitor::::new(self.engine.clone(), self.sm.clone(), visitor); let btree_err = w.walk(&mut path, &v, root).map_err(|e| ArrayError::BTreeError(e)); let mut array_errs = v.array_errs.into_inner().unwrap(); diff --git a/src/pdata/bitset.rs b/src/pdata/bitset.rs index 013d9ec..82bf633 100644 --- a/src/pdata/bitset.rs +++ b/src/pdata/bitset.rs @@ -4,6 +4,7 @@ use std::sync::{Arc, Mutex}; use crate::io_engine::IoEngine; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker}; +use crate::pdata::space_map::*; pub struct CheckedBitSet { bits: FixedBitSet, @@ -77,7 +78,7 @@ pub fn read_bitset( nr_entries: usize, ignore_none_fatal: bool, )-> (CheckedBitSet, Option) { - let w = ArrayWalker::new(engine.clone(), ignore_none_fatal); + let w = ArrayWalker::new(engine, ignore_none_fatal); let mut v = BitsetVisitor::new(nr_entries); let err = w.walk(&mut v, root); let e = match err { @@ -86,3 +87,21 @@ pub fn read_bitset( }; return (v.get_bitset(), e); } + +// TODO: multi-threaded is possible +pub fn read_bitset_with_sm( + engine: Arc, + root: u64, + nr_entries: usize, + sm: Arc>, + ignore_none_fatal: bool, +)-> array::Result<(CheckedBitSet, Option)> { + let w = ArrayWalker::new_with_sm(engine, sm, ignore_none_fatal)?; + let mut v = BitsetVisitor::new(nr_entries); + let err = w.walk(&mut v, root); + let e = match err { + Ok(()) => None, + Err(e) => Some(e), + }; + return Ok((v.get_bitset(), e)); +}