From b7132440d05ec03c07d18020291b9c32cde24a8a Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 18 Aug 2021 10:00:20 +0800 Subject: [PATCH 1/7] [space_map (rust)] Encapsulate implementations --- src/cache/restore.rs | 7 ++----- src/pdata/space_map_common.rs | 7 +++++++ src/thin/restore.rs | 22 ++++------------------ 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/cache/restore.rs b/src/cache/restore.rs index 7997e94..45000a1 100644 --- a/src/cache/restore.rs +++ b/src/cache/restore.rs @@ -2,7 +2,6 @@ use anyhow::Result; use std::convert::TryInto; use std::fs::OpenOptions; -use std::io::Cursor; use std::path::Path; use std::sync::Arc; @@ -14,8 +13,8 @@ use crate::cache::xml; use crate::io_engine::*; use crate::math::*; use crate::pdata::array_builder::*; +use crate::pdata::space_map_common::pack_root; use crate::pdata::space_map_metadata::*; -use crate::pdata::unpack::Pack; use crate::report::*; use crate::write_batcher::*; @@ -240,10 +239,8 @@ impl<'a> MetadataVisitor for Restorer<'a> { //------------------------------------------ fn build_metadata_sm(w: &mut WriteBatcher) -> Result> { - let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; - let mut cur = Cursor::new(&mut sm_root); let r = write_metadata_sm(w)?; - r.pack(&mut cur)?; + let sm_root = pack_root(&r, SPACE_MAP_ROOT_SIZE)?; Ok(sm_root) } diff --git a/src/pdata/space_map_common.rs b/src/pdata/space_map_common.rs index 72c0297..6d86090 100644 --- a/src/pdata/space_map_common.rs +++ b/src/pdata/space_map_common.rs @@ -195,6 +195,13 @@ impl Pack for SMRoot { } } +pub fn pack_root(root: &SMRoot, size: usize) -> Result> { + let mut sm_root = vec![0u8; size]; + let mut cur = Cursor::new(&mut sm_root); + root.pack(&mut cur)?; + Ok(sm_root) +} + //------------------------------------------ pub fn write_common(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result<(Vec, u64)> { diff --git a/src/thin/restore.rs b/src/thin/restore.rs index a1fbb6a..078a3e3 100644 --- a/src/thin/restore.rs +++ b/src/thin/restore.rs @@ -2,7 +2,6 @@ use anyhow::{anyhow, Result}; use std::collections::BTreeMap; use std::fs::OpenOptions; -use std::io::Cursor; use std::ops::Deref; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -10,10 +9,9 @@ use std::sync::{Arc, Mutex}; use crate::io_engine::*; use crate::pdata::btree_builder::*; use crate::pdata::space_map::*; -use crate::pdata::space_map_common::SMRoot; +use crate::pdata::space_map_common::pack_root; use crate::pdata::space_map_disk::*; use crate::pdata::space_map_metadata::*; -use crate::pdata::unpack::Pack; use crate::report::*; use crate::thin::block_time::*; use crate::thin::device_detail::*; @@ -160,7 +158,8 @@ impl<'a> Restorer<'a> { let data_sm_root = build_data_sm(self.w, data_sm.lock().unwrap().deref())?; // Build metadata space map - let (metadata_sm, metadata_sm_root) = build_metadata_sm(self.w)?; + let metadata_sm = write_metadata_sm(self.w)?; + let metadata_sm_root = pack_root(&metadata_sm, SPACE_MAP_ROOT_SIZE)?; // Write the superblock let sb = self.sb.as_ref().unwrap(); @@ -289,25 +288,12 @@ impl<'a> MetadataVisitor for Restorer<'a> { /// Writes a data space map to disk. Returns the space map root that needs /// to be written to the superblock. fn build_data_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result> { - let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; - let mut cur = Cursor::new(&mut sm_root); let r = write_disk_sm(w, sm)?; - r.pack(&mut cur)?; + let sm_root = pack_root(&r, SPACE_MAP_ROOT_SIZE)?; Ok(sm_root) } -/// Writes the metadata space map to disk. Returns the space map root that needs -/// to be written to the superblock. -fn build_metadata_sm(w: &mut WriteBatcher) -> Result<(SMRoot, Vec)> { - let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; - let mut cur = Cursor::new(&mut sm_root); - let r = write_metadata_sm(w)?; - r.pack(&mut cur)?; - - Ok((r, sm_root)) -} - //------------------------------------------ pub struct ThinRestoreOptions<'a> { From 5dad1097c38c7e3e16bf235276278afd6768f494 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 18 Aug 2021 11:14:51 +0800 Subject: [PATCH 2/7] [thin_repair (rust)] Add searching for missing roots Based on the method of commit 9e20465 --- src/pdata/btree_walker.rs | 54 ++- src/thin/metadata_repair.rs | 718 ++++++++++++++++++++++++++++++++++++ src/thin/mod.rs | 1 + 3 files changed, 772 insertions(+), 1 deletion(-) create mode 100644 src/thin/metadata_repair.rs diff --git a/src/pdata/btree_walker.rs b/src/pdata/btree_walker.rs index 801f1cd..b6dd6b7 100644 --- a/src/pdata/btree_walker.rs +++ b/src/pdata/btree_walker.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; use threadpool::ThreadPool; @@ -565,6 +565,58 @@ pub fn btree_to_map_with_path( //------------------------------------------ +struct KeyCollector { + keys: Mutex>, +} + +impl KeyCollector { + fn new() -> KeyCollector { + KeyCollector { + keys: Mutex::new(BTreeSet::new()), + } + } +} + +impl NodeVisitor for KeyCollector { + fn visit( + &self, + _path: &[u64], + _kr: &KeyRange, + _h: &NodeHeader, + keys: &[u64], + _values: &[V], + ) -> Result<()> { + let mut keyset = self.keys.lock().unwrap(); + for k in keys { + keyset.insert(*k); + } + + Ok(()) + } + + fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> { + Ok(()) + } + + fn end_walk(&self) -> Result<()> { + Ok(()) + } +} + +pub fn btree_to_key_set( + path: &mut Vec, + engine: Arc, + ignore_non_fatal: bool, + root: u64, +) -> Result> { + let walker = BTreeWalker::new(engine, ignore_non_fatal); + let visitor = KeyCollector::new(); + walker.walk::<_, V>(path, &visitor, root)?; + Ok(visitor.keys.into_inner().unwrap()) +} + +//------------------------------------------ + struct NoopVisitor { dummy: std::marker::PhantomData, } diff --git a/src/thin/metadata_repair.rs b/src/thin/metadata_repair.rs new file mode 100644 index 0000000..accc4d3 --- /dev/null +++ b/src/thin/metadata_repair.rs @@ -0,0 +1,718 @@ +use anyhow::{anyhow, Result}; +use fixedbitset::FixedBitSet; +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::sync::Arc; + +use crate::checksum; +use crate::io_engine::IoEngine; +use crate::pdata::btree::*; +use crate::pdata::btree_walker::*; +use crate::pdata::space_map_common::*; +use crate::pdata::unpack::Unpack; +use crate::thin::block_time::*; +use crate::thin::device_detail::*; +use crate::thin::superblock::*; + +//------------------------------------------ + +pub struct SuperblockOverrides { + pub transaction_id: Option, + pub data_block_size: Option, + pub nr_data_blocks: Option, +} + +pub struct FoundRoots { + mapping_root: u64, + details_root: u64, + time: u32, +} + +fn merge_time_counts(lhs: &mut BTreeMap, rhs: &BTreeMap) -> Result<()> { + for (t, c) in rhs.iter() { + *lhs.entry(*t).or_insert(0) += c; + } + Ok(()) +} + +struct DevInfo { + b: u64, + nr_devices: u64, + nr_mappings: u64, + key_low: u64, // min dev_id + key_high: u64, // max dev_id, inclusive + time_counts: BTreeMap, + age: u32, + pushed: bool, +} + +impl DevInfo { + fn new(b: u64) -> DevInfo { + DevInfo { + b, + nr_devices: 0, + nr_mappings: 0, + key_low: 0, + key_high: 0, + time_counts: BTreeMap::::new(), + age: 0, + pushed: false, + } + } + + fn push_child(&mut self, child: &DevInfo) -> Result<()> { + if self.key_high > 0 && child.key_low <= self.key_high { + return Err(anyhow!("incompatible child")); + } + if !self.pushed { + self.key_low = child.key_low; + self.pushed = true; + } + self.key_high = child.key_high; + self.nr_devices += child.nr_devices; + self.nr_mappings += child.nr_mappings; + merge_time_counts(&mut self.time_counts, &child.time_counts)?; + self.age = std::cmp::max(self.age, child.age); + + Ok(()) + } +} + +struct MappingsInfo { + _b: u64, + nr_mappings: u64, + key_low: u64, // min mapped block + key_high: u64, // max mapped block, inclusive + time_counts: BTreeMap, + age: u32, + pushed: bool, +} + +impl MappingsInfo { + fn new(b: u64) -> MappingsInfo { + MappingsInfo { + _b: b, + nr_mappings: 0, + key_low: 0, + key_high: 0, + time_counts: BTreeMap::::new(), + age: 0, + pushed: false, + } + } + + fn push_child(&mut self, child: &MappingsInfo) -> Result<()> { + if self.key_high > 0 && child.key_low <= self.key_high { + return Err(anyhow!("incompatible child")); + } + if !self.pushed { + self.key_low = child.key_low; + self.pushed = true; + } + self.key_high = child.key_high; + self.nr_mappings += child.nr_mappings; + merge_time_counts(&mut self.time_counts, &child.time_counts)?; + self.age = std::cmp::max(self.age, child.age); + + Ok(()) + } +} + +struct DetailsInfo { + _b: u64, + nr_devices: u64, + nr_mappings: u64, + key_low: u64, + key_high: u64, // inclusive + age: u32, + pushed: bool, +} + +impl DetailsInfo { + fn new(b: u64) -> DetailsInfo { + DetailsInfo { + _b: b, + nr_devices: 0, + nr_mappings: 0, + key_low: 0, + key_high: 0, + age: 0, + pushed: false, + } + } + + fn push_child(&mut self, child: &DetailsInfo) -> Result<()> { + if self.key_high > 0 && child.key_low <= self.key_high { + return Err(anyhow!("incompatible child")); + } + if !self.pushed { + self.key_low = child.key_low; + self.pushed = true; + } + self.key_high = child.key_high; + self.nr_devices += child.nr_devices; + self.nr_mappings += child.nr_mappings; + self.age = std::cmp::max(self.age, child.age); + + Ok(()) + } +} + +enum NodeInfo { + Dev(DevInfo), + Mappings(MappingsInfo), + Details(DetailsInfo), +} + +struct NodeCollector { + engine: Arc, + nr_blocks: u64, + examined: FixedBitSet, + referenced: FixedBitSet, + infos: BTreeMap, +} + +impl NodeCollector { + pub fn new(engine: Arc) -> NodeCollector { + let nr_blocks = engine.get_nr_blocks(); + NodeCollector { + engine, + nr_blocks, + examined: FixedBitSet::with_capacity(nr_blocks as usize), + referenced: FixedBitSet::with_capacity(nr_blocks as usize), + infos: BTreeMap::::new(), + } + } + + fn gather_dev_subtree_info( + &mut self, + header: &NodeHeader, + _keys: &[u64], + values: &[u64], + ) -> Result { + let mut info = DevInfo::new(header.block); + + for b in values { + let child = self.get_info(*b)?; + if let NodeInfo::Dev(ref child) = child { + info.push_child(child)?; + } else { + return Err(anyhow!("incompatible child type")); + } + } + + Ok(NodeInfo::Dev(info)) + } + + fn gather_mappings_subtree_info( + &mut self, + header: &NodeHeader, + _keys: &[u64], + values: &[u64], + ) -> Result { + let mut info = MappingsInfo::new(header.block); + + for b in values { + let child = self.get_info(*b)?; + if let NodeInfo::Mappings(ref child) = child { + info.push_child(child)?; + } else { + return Err(anyhow!("incompatible child type")); + } + } + + Ok(NodeInfo::Mappings(info)) + } + + fn gather_details_subtree_info( + &mut self, + header: &NodeHeader, + _keys: &[u64], + values: &[u64], + ) -> Result { + let mut info = DetailsInfo::new(header.block); + + for b in values { + let child = self.get_info(*b)?; + if let NodeInfo::Details(ref child) = child { + info.push_child(child)?; + } else { + return Err(anyhow!("incompatible child type")); + } + } + + Ok(NodeInfo::Details(info)) + } + + fn gather_subtree_info( + &mut self, + header: &NodeHeader, + keys: &[u64], + values: &[u64], + ) -> Result { + // An internal node should have at least one entry + if header.nr_entries == 0 { + return Err(anyhow!("unexpected nr_entries")); + } + + let first_child = self.get_info(values[0])?; + let info = match first_child { + NodeInfo::Dev { .. } => self.gather_dev_subtree_info(header, keys, values), + NodeInfo::Mappings { .. } => self.gather_mappings_subtree_info(header, keys, values), + NodeInfo::Details { .. } => self.gather_details_subtree_info(header, keys, values), + }?; + + // the node is a valid parent, so mark the children as non-orphan + for b in values { + self.referenced.set(*b as usize, true); + } + + Ok(info) + } + + fn gather_dev_leaf_info( + &mut self, + header: &NodeHeader, + keys: &[u64], + values: &[u64], + ) -> Result { + let mut info = DevInfo::new(header.block); + info.nr_devices = header.nr_entries as u64; + + // min & max device id + if header.nr_entries > 0 { + info.key_low = keys[0]; + info.key_high = keys[keys.len() - 1]; + } + + for b in values { + let child = self.get_info(*b)?; // subtree root + if let NodeInfo::Mappings(ref child) = child { + info.nr_mappings += child.nr_mappings; + merge_time_counts(&mut info.time_counts, &child.time_counts)?; + info.age = std::cmp::max(info.age, child.age); + } else { + return Err(anyhow!("not a data mapping subtree root")); + } + } + + Ok(NodeInfo::Dev(info)) + } + + fn gather_mapping_leaf_info( + &self, + header: &NodeHeader, + keys: &[u64], + values: &[BlockTime], + ) -> Result { + let mut info = MappingsInfo::new(header.block); + info.nr_mappings = header.nr_entries as u64; + + // min & max logical block address + if header.nr_entries > 0 { + info.key_low = keys[0]; + info.key_high = keys[keys.len() - 1]; + } + + for bt in values { + *info.time_counts.entry(bt.time).or_insert(0) += 1; + info.age = std::cmp::max(info.age, bt.time); + } + + // TODO: propagate the low & high data block address upward, + // for rebuilding the nr_data_blocks in superblock? + Ok(NodeInfo::Mappings(info)) + } + + fn gather_details_leaf_info( + &self, + header: &NodeHeader, + keys: &[u64], + values: &[DeviceDetail], + ) -> Result { + let mut info = DetailsInfo::new(header.block); + info.nr_devices = header.nr_entries as u64; + + // min & max device id + if header.nr_entries > 0 { + info.key_low = keys[0]; + info.key_high = keys[keys.len() - 1]; + } + + for details in values { + info.nr_mappings += details.mapped_blocks; + info.age = std::cmp::max(info.age, details.creation_time); + info.age = std::cmp::max(info.age, details.snapshotted_time); + } + + Ok(NodeInfo::Details(info)) + } + + fn is_top_level(&self, values: &[u64]) -> bool { + if values.is_empty() { + return false; + } + + for v in values { + if *v > self.nr_blocks { + return false; + } + } + + true + } + + fn gather_info(&mut self, b: u64) -> Result { + let blk = self.engine.read(b)?; + + let bt = checksum::metadata_block_type(blk.get_data()); + if bt != checksum::BT::NODE { + return Err(anyhow!("not a btree node")); + } + + let (_, hdr) = NodeHeader::unpack(blk.get_data())?; + if hdr.value_size as usize == std::mem::size_of::() { + let node = unpack_node::(&[0], blk.get_data(), true, true)?; + match node { + Node::Internal { + ref header, + ref keys, + ref values, + } => self.gather_subtree_info(header, keys, values), + Node::Leaf { + ref header, + ref keys, + ref values, + } => { + if self.is_top_level(values) { + self.gather_dev_leaf_info(header, keys, values) + } else { + // FIXME: convert the values only, to avoid unpacking the node twice + let node = unpack_node::(&[0], blk.get_data(), true, true)?; + if let Node::Leaf { + ref header, + ref keys, + ref values, + } = node + { + self.gather_mapping_leaf_info(header, keys, values) + } else { + Err(anyhow!("unexpected internal node")) + } + } + } + } + } else if hdr.value_size == DeviceDetail::disk_size() { + let node = unpack_node::(&[0], blk.get_data(), true, true)?; + if let Node::Leaf { + ref header, + ref keys, + ref values, + } = node + { + self.gather_details_leaf_info(header, keys, values) + } else { + Err(anyhow!("unexpected value size within an internal node")) + } + } else { + Err(anyhow!("not the value size of interest")) + } + } + + fn read_info(&self, b: u64) -> Result<&NodeInfo> { + if self.examined.contains(b as usize) { + // TODO: use an extra 'valid' bitset for faster lookup? + self.infos + .get(&b) + .ok_or_else(|| anyhow!("block {} was examined as invalid", b)) + } else { + Err(anyhow!("not examined")) + } + } + + fn get_info(&mut self, b: u64) -> Result<&NodeInfo> { + if self.examined.contains(b as usize) { + // TODO: use an extra 'valid' bitset for faster lookup? + self.infos + .get(&b) + .ok_or_else(|| anyhow!("block {} was examined as invalid", b)) + } else { + self.examined.set(b as usize, true); + let info = self.gather_info(b)?; + Ok(self.infos.entry(b).or_insert(info)) + } + } + + fn collect_infos(&mut self) -> Result<()> { + for b in 0..self.nr_blocks { + let _ret = self.get_info(b); + } + Ok(()) + } + + fn gather_roots(&self) -> Result<(Vec, Vec)> { + let mut dev_roots = Vec::::new(); + let mut details_roots = Vec::::new(); + + for b in 0..self.nr_blocks { + // skip non-root blocks + if self.referenced.contains(b as usize) { + continue; + } + + // skip blocks we're not interested in + let info = self.read_info(b as u64); + if info.is_err() { + continue; + } + let info = info.unwrap(); + + match info { + NodeInfo::Dev(_) => dev_roots.push(b), + NodeInfo::Details(_) => details_roots.push(b), + _ => {} + }; + } + + Ok((dev_roots, details_roots)) + } + + fn find_roots_with_compatible_ids( + &mut self, + dev_roots: &[u64], + details_roots: &[u64], + ) -> Result> { + let mut root_pairs = Vec::<(u64, u64)>::new(); + + for dev_root in dev_roots { + let mut path = vec![0]; + let ids1 = btree_to_key_set::(&mut path, self.engine.clone(), true, *dev_root)?; + + for details_root in details_roots { + let mut path = vec![0]; + let ids2 = btree_to_key_set::( + &mut path, + self.engine.clone(), + true, + *details_root, + )?; + + if ids1 != ids2 { + continue; + } + + root_pairs.push((*dev_root, *details_root)); + } + } + + Ok(root_pairs) + } + + fn filter_roots_with_incompatible_mapped_blocks( + &self, + root_pairs: &[(u64, u64)], + ) -> Result> { + let mut filtered = Vec::<(u64, u64)>::new(); + + for (dev_root, details_root) in root_pairs { + let dev_info; + if let NodeInfo::Dev(i) = self.read_info(*dev_root)? { + dev_info = i; + } else { + continue; + } + + let details_info; + if let NodeInfo::Details(i) = self.read_info(*details_root)? { + details_info = i; + } else { + continue; + } + + // FIXME: compare the ages + if dev_info.nr_devices != details_info.nr_devices + || dev_info.nr_mappings != details_info.nr_mappings + { + continue; + } + + filtered.push((*dev_root, *details_root)); + } + + Ok(filtered) + } + + fn compare_time_counts(lhs: &BTreeMap, rhs: &BTreeMap) -> Ordering { + let mut lhs_it = lhs.iter().rev(); + let mut rhs_it = rhs.iter().rev(); + let mut lhs_end = false; + let mut rhs_end = false; + + while !lhs_end && !rhs_end { + if let Some((lhs_time, lhs_count)) = lhs_it.next() { + if let Some((rhs_time, rhs_count)) = rhs_it.next() { + if lhs_time > rhs_time { + return Ordering::Less; + } else if rhs_time > lhs_time { + return Ordering::Greater; + } else if lhs_count > rhs_count { + return Ordering::Less; + } else if rhs_count > lhs_count { + return Ordering::Greater; + } + } else { + rhs_end = true; + } + } else { + lhs_end = true; + } + } + + if lhs_end && !rhs_end { + Ordering::Less + } else if !lhs_end && rhs_end { + Ordering::Greater + } else { + Ordering::Equal + } + } + + fn compare_roots(lhs: &(&DevInfo, u64), rhs: &(&DevInfo, u64)) -> Ordering { + // TODO: sort by other criteria? + Self::compare_time_counts(&lhs.0.time_counts, &rhs.0.time_counts) + } + + fn sort_roots(&self, root_pairs: &[(u64, u64)]) -> Result> { + let mut infos = Vec::<(&DevInfo, u64)>::new(); + + for (dev_root, details_root) in root_pairs { + let dev_info; + if let NodeInfo::Dev(i) = self.read_info(*dev_root)? { + dev_info = i; + } else { + continue; + } + + infos.push((dev_info, *details_root)); + } + + infos.sort_by(Self::compare_roots); + + let mut sorted = Vec::<(u64, u64)>::new(); + for (dev_info, details_root) in infos { + sorted.push((dev_info.b, details_root)); + } + + Ok(sorted) + } + + fn find_root_pairs( + &mut self, + dev_roots: &[u64], + details_roots: &[u64], + ) -> Result> { + let pairs = self.find_roots_with_compatible_ids(dev_roots, details_roots)?; + let pairs = self.filter_roots_with_incompatible_mapped_blocks(&pairs)?; + self.sort_roots(&pairs) + } + + fn to_found_roots(&self, dev_root: u64, details_root: u64) -> Result { + let dev_info; + if let NodeInfo::Dev(i) = self.read_info(dev_root)? { + dev_info = i; + } else { + return Err(anyhow!("not a top-level root")); + } + + let details_info; + if let NodeInfo::Details(i) = self.read_info(details_root)? { + details_info = i; + } else { + return Err(anyhow!("not a details root")); + } + + Ok(FoundRoots { + mapping_root: dev_root, + details_root, + time: std::cmp::max(dev_info.age, details_info.age), + }) + } + + pub fn find_roots(mut self) -> Result { + self.collect_infos()?; + let (dev_roots, details_roots) = self.gather_roots()?; + let pairs = self.find_root_pairs(&dev_roots, &details_roots)?; + + if pairs.is_empty() { + return Err(anyhow!("no compatible roots found")); + } + + self.to_found_roots(pairs[0].0, pairs[0].1) + } +} + +//------------------------------------------ + +pub fn is_superblock_consistent( + sb: Superblock, + engine: Arc, +) -> Result { + let mut path = vec![0]; + let ids1 = btree_to_key_set::(&mut path, engine.clone(), true, sb.mapping_root)?; + + path = vec![0]; + let ids2 = btree_to_key_set::(&mut path, engine.clone(), true, sb.details_root)?; + + if ids1 != ids2 { + return Err(anyhow!("inconsistent device ids")); + } + + Ok(sb) +} + +pub fn rebuild_superblock( + engine: Arc, + opts: &SuperblockOverrides, +) -> Result { + // Check parameters + let nr_data_blocks = opts.nr_data_blocks.ok_or_else(|| anyhow!(""))?; + let transaction_id = opts.transaction_id.ok_or_else(|| anyhow!(""))?; + let data_block_size = opts.data_block_size.ok_or_else(|| anyhow!(""))?; + + let c = NodeCollector::new(engine.clone()); + let roots = c.find_roots()?; + let sm_root = SMRoot { + nr_blocks: nr_data_blocks, + nr_allocated: 0, + bitmap_root: 0, + ref_count_root: 0, + }; + let data_sm_root = pack_root(&sm_root, SPACE_MAP_ROOT_SIZE)?; + + Ok(Superblock { + flags: SuperblockFlags { needs_check: false }, + block: SUPERBLOCK_LOCATION, + version: 2, + time: roots.time, + transaction_id, + metadata_snap: 0, + data_sm_root, + metadata_sm_root: vec![0u8; SPACE_MAP_ROOT_SIZE], + mapping_root: roots.mapping_root, + details_root: roots.details_root, + data_block_size, + nr_metadata_blocks: 0, + }) +} + +pub fn read_or_rebuild_superblock( + engine: Arc, + loc: u64, + opts: &SuperblockOverrides, +) -> Result { + read_superblock(engine.as_ref(), loc) + .and_then(|sb| is_superblock_consistent(sb, engine.clone())) + .or_else(|_| rebuild_superblock(engine, &opts)) +} + +//------------------------------------------ diff --git a/src/thin/mod.rs b/src/thin/mod.rs index 1f44e9e..197f2c2 100644 --- a/src/thin/mod.rs +++ b/src/thin/mod.rs @@ -4,6 +4,7 @@ pub mod device_detail; pub mod dump; pub mod ir; pub mod metadata; +pub mod metadata_repair; pub mod repair; pub mod restore; pub mod runs; From 434d24d10a61488c934d8bcce136ab22fbad37c8 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Mon, 23 Aug 2021 17:23:05 +0800 Subject: [PATCH 3/7] [thin_repair (rust)] Support setting missing superblock fields automatically While rebuilding a superblock, the transaction_id, data_block_size, and nr_data_blocks could be inherited from the current superblock if available (assumed it's a multiple-activated copy, so supposed partially valid), or derived from the probed data mappings and device details, that saves the hassle of manual input. Note that either the current superblock or user-overrides must be compatible with the probed values, thus the filters are applied. --- src/thin/metadata_repair.rs | 106 +++++++++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/src/thin/metadata_repair.rs b/src/thin/metadata_repair.rs index accc4d3..43f095c 100644 --- a/src/thin/metadata_repair.rs +++ b/src/thin/metadata_repair.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use std::cmp::Ordering; use std::collections::BTreeMap; +use std::fmt; use std::sync::Arc; use crate::checksum; @@ -26,6 +27,8 @@ pub struct FoundRoots { mapping_root: u64, details_root: u64, time: u32, + transaction_id: u64, + nr_data_blocks: u64, } fn merge_time_counts(lhs: &mut BTreeMap, rhs: &BTreeMap) -> Result<()> { @@ -43,6 +46,7 @@ struct DevInfo { key_high: u64, // max dev_id, inclusive time_counts: BTreeMap, age: u32, + highest_mapped_data_block: u64, pushed: bool, } @@ -56,6 +60,7 @@ impl DevInfo { key_high: 0, time_counts: BTreeMap::::new(), age: 0, + highest_mapped_data_block: 0, pushed: false, } } @@ -73,6 +78,10 @@ impl DevInfo { self.nr_mappings += child.nr_mappings; merge_time_counts(&mut self.time_counts, &child.time_counts)?; self.age = std::cmp::max(self.age, child.age); + self.highest_mapped_data_block = std::cmp::max( + self.highest_mapped_data_block, + child.highest_mapped_data_block, + ); Ok(()) } @@ -85,6 +94,7 @@ struct MappingsInfo { key_high: u64, // max mapped block, inclusive time_counts: BTreeMap, age: u32, + highest_mapped_data_block: u64, pushed: bool, } @@ -97,6 +107,7 @@ impl MappingsInfo { key_high: 0, time_counts: BTreeMap::::new(), age: 0, + highest_mapped_data_block: 0, pushed: false, } } @@ -113,6 +124,10 @@ impl MappingsInfo { self.nr_mappings += child.nr_mappings; merge_time_counts(&mut self.time_counts, &child.time_counts)?; self.age = std::cmp::max(self.age, child.age); + self.highest_mapped_data_block = std::cmp::max( + self.highest_mapped_data_block, + child.highest_mapped_data_block, + ); Ok(()) } @@ -122,8 +137,9 @@ struct DetailsInfo { _b: u64, nr_devices: u64, nr_mappings: u64, - key_low: u64, - key_high: u64, // inclusive + key_low: u64, // min dev_id + key_high: u64, // max dev_id, inclusive + max_tid: u64, age: u32, pushed: bool, } @@ -136,6 +152,7 @@ impl DetailsInfo { nr_mappings: 0, key_low: 0, key_high: 0, + max_tid: 0, age: 0, pushed: false, } @@ -152,6 +169,7 @@ impl DetailsInfo { self.key_high = child.key_high; self.nr_devices += child.nr_devices; self.nr_mappings += child.nr_mappings; + self.max_tid = std::cmp::max(self.max_tid, child.max_tid); self.age = std::cmp::max(self.age, child.age); Ok(()) @@ -291,6 +309,10 @@ impl NodeCollector { info.nr_mappings += child.nr_mappings; merge_time_counts(&mut info.time_counts, &child.time_counts)?; info.age = std::cmp::max(info.age, child.age); + info.highest_mapped_data_block = std::cmp::max( + info.highest_mapped_data_block, + child.highest_mapped_data_block, + ); } else { return Err(anyhow!("not a data mapping subtree root")); } @@ -315,12 +337,12 @@ impl NodeCollector { } for bt in values { + info.highest_mapped_data_block = + std::cmp::max(bt.block, info.highest_mapped_data_block); *info.time_counts.entry(bt.time).or_insert(0) += 1; info.age = std::cmp::max(info.age, bt.time); } - // TODO: propagate the low & high data block address upward, - // for rebuilding the nr_data_blocks in superblock? Ok(NodeInfo::Mappings(info)) } @@ -341,6 +363,7 @@ impl NodeCollector { for details in values { info.nr_mappings += details.mapped_blocks; + info.max_tid = std::cmp::max(info.max_tid, details.transaction_id); info.age = std::cmp::max(info.age, details.creation_time); info.age = std::cmp::max(info.age, details.snapshotted_time); } @@ -635,6 +658,8 @@ impl NodeCollector { mapping_root: dev_root, details_root, time: std::cmp::max(dev_info.age, details_info.age), + transaction_id: details_info.max_tid + 1, // tid in superblock is ahead by 1 + nr_data_blocks: dev_info.highest_mapped_data_block + 1, }) } @@ -651,20 +676,43 @@ impl NodeCollector { } } +fn check_data_block_size(bs: u32) -> Result { + if !(128..=2097152).contains(&bs) || (bs & 0x7F != 0) { + return Err(anyhow!("invalid data block size")); + } + Ok(bs) +} + //------------------------------------------ +#[derive(Debug)] +struct SuperblockError { + failed_sb: Option, +} + +impl fmt::Display for SuperblockError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.failed_sb) + } +} + +impl std::error::Error for SuperblockError {} + pub fn is_superblock_consistent( sb: Superblock, engine: Arc, ) -> Result { let mut path = vec![0]; - let ids1 = btree_to_key_set::(&mut path, engine.clone(), true, sb.mapping_root)?; + let ids1 = btree_to_key_set::(&mut path, engine.clone(), true, sb.mapping_root); path = vec![0]; - let ids2 = btree_to_key_set::(&mut path, engine.clone(), true, sb.details_root)?; + let ids2 = btree_to_key_set::(&mut path, engine.clone(), true, sb.details_root); - if ids1 != ids2 { - return Err(anyhow!("inconsistent device ids")); + if ids1.is_err() || ids2.is_err() || ids1.unwrap() != ids2.unwrap() { + return Err(anyhow::Error::new(SuperblockError { + failed_sb: Some(sb), + }) + .context("inconsistent device ids")); } Ok(sb) @@ -672,15 +720,42 @@ pub fn is_superblock_consistent( pub fn rebuild_superblock( engine: Arc, + ref_sb: Option, opts: &SuperblockOverrides, ) -> Result { - // Check parameters - let nr_data_blocks = opts.nr_data_blocks.ok_or_else(|| anyhow!(""))?; - let transaction_id = opts.transaction_id.ok_or_else(|| anyhow!(""))?; - let data_block_size = opts.data_block_size.ok_or_else(|| anyhow!(""))?; + // 1. Takes the user overrides + // 2. Takes the reference if there's no user overrides + // 3. Returns Err if none of the values above are present + // 4. Validates the taken value + let data_block_size = opts + .data_block_size + .or_else(|| ref_sb.as_ref().map(|sb| sb.data_block_size)) + .ok_or_else(|| { + anyhow!("data block size needs to be provided due to corruption in the superblock") + }) + .and_then(check_data_block_size)?; let c = NodeCollector::new(engine.clone()); let roots = c.find_roots()?; + + let transaction_id = opts + .transaction_id + .or_else(|| ref_sb.as_ref().map(|sb| sb.transaction_id)) + .filter(|tid| *tid > roots.transaction_id) + .unwrap_or(roots.transaction_id); + + let nr_data_blocks = opts + .nr_data_blocks + .or_else(|| { + ref_sb.as_ref().and_then(|sb| { + unpack_root(&sb.data_sm_root) + .ok() + .map(|root| root.nr_blocks) + }) + }) + .filter(|n| *n > roots.nr_data_blocks) + .unwrap_or(roots.nr_data_blocks); + let sm_root = SMRoot { nr_blocks: nr_data_blocks, nr_allocated: 0, @@ -712,7 +787,12 @@ pub fn read_or_rebuild_superblock( ) -> Result { read_superblock(engine.as_ref(), loc) .and_then(|sb| is_superblock_consistent(sb, engine.clone())) - .or_else(|_| rebuild_superblock(engine, &opts)) + .or_else(|e| { + let ref_sb = e + .downcast_ref::() + .and_then(|err| err.failed_sb.clone()); + rebuild_superblock(engine, ref_sb, opts) + }) } //------------------------------------------ From 213442bffd84937535e2ca971c792732f947fb1e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 24 Aug 2021 02:17:24 +0800 Subject: [PATCH 4/7] [all (rust)] Fix cosmetic names for argument values --- src/bin/cache_dump.rs | 2 +- src/bin/cache_repair.rs | 4 ++-- src/bin/cache_restore.rs | 4 ++-- src/bin/thin_dump.rs | 2 +- src/bin/thin_repair.rs | 4 ++-- src/bin/thin_restore.rs | 4 ++-- src/bin/thin_shrink.rs | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/bin/cache_dump.rs b/src/bin/cache_dump.rs index 93644ec..5440725 100644 --- a/src/bin/cache_dump.rs +++ b/src/bin/cache_dump.rs @@ -30,7 +30,7 @@ fn main() { .help("Specify the output file rather than stdout") .short("o") .long("output") - .value_name("OUTPUT"), + .value_name("FILE"), ) // arguments .arg( diff --git a/src/bin/cache_repair.rs b/src/bin/cache_repair.rs index d964a0f..f2d4f0c 100644 --- a/src/bin/cache_repair.rs +++ b/src/bin/cache_repair.rs @@ -34,7 +34,7 @@ fn main() { .help("Specify the input device") .short("i") .long("input") - .value_name("INPUT") + .value_name("FILE") .required(true), ) .arg( @@ -42,7 +42,7 @@ fn main() { .help("Specify the output device") .short("o") .long("output") - .value_name("OUTPUT") + .value_name("FILE") .required(true), ); diff --git a/src/bin/cache_restore.rs b/src/bin/cache_restore.rs index f580777..714d2d4 100644 --- a/src/bin/cache_restore.rs +++ b/src/bin/cache_restore.rs @@ -41,7 +41,7 @@ fn main() { .help("Specify the input xml") .short("i") .long("input") - .value_name("INPUT") + .value_name("FILE") .required(true), ) .arg( @@ -49,7 +49,7 @@ fn main() { .help("Specify the output device to check") .short("o") .long("output") - .value_name("OUTPUT") + .value_name("FILE") .required(true), ); diff --git a/src/bin/thin_dump.rs b/src/bin/thin_dump.rs index 606ea42..7533961 100644 --- a/src/bin/thin_dump.rs +++ b/src/bin/thin_dump.rs @@ -52,7 +52,7 @@ fn main() { .help("Specify the output file rather than stdout") .short("o") .long("output") - .value_name("OUTPUT"), + .value_name("FILE"), ) // arguments .arg( diff --git a/src/bin/thin_repair.rs b/src/bin/thin_repair.rs index 7d41bf3..2cac56e 100644 --- a/src/bin/thin_repair.rs +++ b/src/bin/thin_repair.rs @@ -34,7 +34,7 @@ fn main() { .help("Specify the input device") .short("i") .long("input") - .value_name("INPUT") + .value_name("FILE") .required(true), ) .arg( @@ -42,7 +42,7 @@ fn main() { .help("Specify the output device") .short("o") .long("output") - .value_name("OUTPUT") + .value_name("FILE") .required(true), ) .arg( diff --git a/src/bin/thin_restore.rs b/src/bin/thin_restore.rs index 34cfb85..c221326 100644 --- a/src/bin/thin_restore.rs +++ b/src/bin/thin_restore.rs @@ -34,7 +34,7 @@ fn main() { .help("Specify the input xml") .short("i") .long("input") - .value_name("INPUT") + .value_name("FILE") .required(true), ) .arg( @@ -42,7 +42,7 @@ fn main() { .help("Specify the output device") .short("o") .long("output") - .value_name("OUTPUT") + .value_name("FILE") .required(true), ) .arg( diff --git a/src/bin/thin_shrink.rs b/src/bin/thin_shrink.rs index ff7fb45..722ef2b 100644 --- a/src/bin/thin_shrink.rs +++ b/src/bin/thin_shrink.rs @@ -20,7 +20,7 @@ fn main() { .required(true) .short("i") .long("input") - .value_name("INPUT") + .value_name("FILE") .takes_value(true), ) .arg( @@ -29,7 +29,7 @@ fn main() { .required(true) .short("o") .long("output") - .value_name("OUTPUT") + .value_name("FILE") .takes_value(true), ) .arg( From 8a822cebec8c52b6010f91872c5fe537aa3f4299 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 24 Aug 2021 02:18:08 +0800 Subject: [PATCH 5/7] [thin_repair/thin_dump (rust)] Support rebuilding superblock --- src/bin/thin_dump.rs | 46 ++++++++++++++++++++++++++++++++++++++++ src/bin/thin_repair.rs | 48 +++++++++++++++++++++++++++++++++++++----- src/thin/dump.rs | 10 ++++++++- src/thin/repair.rs | 5 ++++- 4 files changed, 102 insertions(+), 7 deletions(-) diff --git a/src/bin/thin_dump.rs b/src/bin/thin_dump.rs index 7533961..4f6c07d 100644 --- a/src/bin/thin_dump.rs +++ b/src/bin/thin_dump.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use thinp::file_utils; use thinp::report::*; use thinp::thin::dump::{dump, ThinDumpOptions}; +use thinp::thin::metadata_repair::SuperblockOverrides; fn main() { let parser = App::new("thin_dump") @@ -40,6 +41,12 @@ fn main() { .long("skip-mappings"), ) // options + .arg( + Arg::with_name("DATA_BLOCK_SIZE") + .help("Provide the data block size for repairing") + .long("data-block-size") + .value_name("SECTORS"), + ) .arg( Arg::with_name("METADATA_SNAPSHOT") .help("Access the metadata snapshot on a live pool") @@ -47,6 +54,12 @@ fn main() { .long("metadata-snapshot") .value_name("METADATA_SNAPSHOT"), ) + .arg( + Arg::with_name("NR_DATA_BLOCKS") + .help("Override the number of data blocks if needed") + .long("nr-data-blocks") + .value_name("NUM"), + ) .arg( Arg::with_name("OUTPUT") .help("Specify the output file rather than stdout") @@ -54,6 +67,12 @@ fn main() { .long("output") .value_name("FILE"), ) + .arg( + Arg::with_name("TRANSACTION_ID") + .help("Override the transaction id if needed") + .long("transaction-id") + .value_name("NUM"), + ) // arguments .arg( Arg::with_name("INPUT") @@ -75,6 +94,27 @@ fn main() { exit(1); } + let transaction_id = matches.value_of("TRANSACTION_ID").map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse transaction_id"); + exit(1); + }) + }); + + let data_block_size = matches.value_of("DATA_BLOCK_SIZE").map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse data_block_size"); + exit(1); + }) + }); + + let nr_data_blocks = matches.value_of("NR_DATA_BLOCKS").map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse nr_data_blocks"); + exit(1); + }) + }); + let report; if matches.is_present("QUIET") { @@ -90,6 +130,12 @@ fn main() { output: output_file, async_io: matches.is_present("ASYNC_IO"), report, + repair: matches.is_present("REPAIR"), + overrides: SuperblockOverrides { + transaction_id, + data_block_size, + nr_data_blocks, + }, }; if let Err(reason) = dump(opts) { diff --git a/src/bin/thin_repair.rs b/src/bin/thin_repair.rs index 2cac56e..781cb31 100644 --- a/src/bin/thin_repair.rs +++ b/src/bin/thin_repair.rs @@ -9,6 +9,7 @@ use std::process::exit; use std::sync::Arc; use thinp::file_utils; use thinp::report::*; +use thinp::thin::metadata_repair::SuperblockOverrides; use thinp::thin::repair::{repair, ThinRepairOptions}; fn main() { @@ -29,6 +30,12 @@ fn main() { .long("quiet"), ) // options + .arg( + Arg::with_name("DATA_BLOCK_SIZE") + .help("Provide the data block size for repairing") + .long("data-block-size") + .value_name("SECTORS"), + ) .arg( Arg::with_name("INPUT") .help("Specify the input device") @@ -37,6 +44,12 @@ fn main() { .value_name("FILE") .required(true), ) + .arg( + Arg::with_name("NR_DATA_BLOCKS") + .help("Override the number of data blocks if needed") + .long("nr-data-blocks") + .value_name("NUM"), + ) .arg( Arg::with_name("OUTPUT") .help("Specify the output device") @@ -46,11 +59,10 @@ fn main() { .required(true), ) .arg( - Arg::with_name("OVERRIDE_MAPPING_ROOT") - .help("Specify a mapping root to use") - .long("override-mapping-root") - .value_name("OVERRIDE_MAPPING_ROOT") - .takes_value(true), + Arg::with_name("TRANSACTION_ID") + .help("Override the transaction id if needed") + .long("transaction-id") + .value_name("NUM"), ); let matches = parser.get_matches(); @@ -62,6 +74,27 @@ fn main() { exit(1); } + let transaction_id = matches.value_of("TRANSACTION_ID").map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse transaction_id"); + exit(1); + }) + }); + + let data_block_size = matches.value_of("DATA_BLOCK_SIZE").map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse data_block_size"); + exit(1); + }) + }); + + let nr_data_blocks = matches.value_of("NR_DATA_BLOCKS").map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse nr_data_blocks"); + exit(1); + }) + }); + let report; if matches.is_present("QUIET") { @@ -77,6 +110,11 @@ fn main() { output: &output_file, async_io: matches.is_present("ASYNC_IO"), report, + overrides: SuperblockOverrides { + transaction_id, + data_block_size, + nr_data_blocks, + }, }; if let Err(reason) = repair(opts) { diff --git a/src/thin/dump.rs b/src/thin/dump.rs index f588c5d..7127c16 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -15,6 +15,7 @@ use crate::report::*; use crate::thin::block_time::*; use crate::thin::ir::{self, MetadataVisitor}; use crate::thin::metadata::*; +use crate::thin::metadata_repair::*; use crate::thin::superblock::*; use crate::thin::xml; @@ -147,6 +148,8 @@ pub struct ThinDumpOptions<'a> { pub output: Option<&'a Path>, pub async_io: bool, pub report: Arc, + pub repair: bool, + pub overrides: SuperblockOverrides, } struct Context { @@ -311,7 +314,12 @@ pub fn dump_metadata( pub fn dump(opts: ThinDumpOptions) -> Result<()> { let ctx = mk_context(&opts)?; - let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; + let sb; + if opts.repair { + sb = read_or_rebuild_superblock(ctx.engine.clone(), SUPERBLOCK_LOCATION, &opts.overrides)?; + } else { + sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; + } let md = build_metadata(ctx.engine.clone(), &sb)?; ctx.report diff --git a/src/thin/repair.rs b/src/thin/repair.rs index 4f74371..dd5b996 100644 --- a/src/thin/repair.rs +++ b/src/thin/repair.rs @@ -7,6 +7,7 @@ use crate::pdata::space_map_metadata::*; use crate::report::*; use crate::thin::dump::*; use crate::thin::metadata::*; +use crate::thin::metadata_repair::*; use crate::thin::restore::*; use crate::thin::superblock::*; use crate::write_batcher::*; @@ -18,6 +19,7 @@ pub struct ThinRepairOptions<'a> { pub output: &'a Path, pub async_io: bool, pub report: Arc, + pub overrides: SuperblockOverrides, } struct Context { @@ -53,7 +55,8 @@ fn new_context(opts: &ThinRepairOptions) -> Result { pub fn repair(opts: ThinRepairOptions) -> Result<()> { let ctx = new_context(&opts)?; - let sb = read_superblock(ctx.engine_in.as_ref(), SUPERBLOCK_LOCATION)?; + let sb = + read_or_rebuild_superblock(ctx.engine_in.clone(), SUPERBLOCK_LOCATION, &opts.overrides)?; let md = build_metadata(ctx.engine_in.clone(), &sb)?; let md = optimise_metadata(md)?; From 71d47ef58bffdfcb4f7748502fe1a3433eb697a8 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 27 Aug 2021 11:17:27 +0800 Subject: [PATCH 6/7] [thin/cache (rust)] Validate superblock checksum --- src/cache/superblock.rs | 6 +++++- src/thin/superblock.rs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cache/superblock.rs b/src/cache/superblock.rs index 23bc474..abe9b5d 100644 --- a/src/cache/superblock.rs +++ b/src/cache/superblock.rs @@ -134,7 +134,11 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result { let b = engine.read(loc)?; - if let Ok((_, sb)) = unpack(&b.get_data()) { + if metadata_block_type(b.get_data()) != BT::CACHE_SUPERBLOCK { + return Err(anyhow!("bad checksum in superblock")); + } + + if let Ok((_, sb)) = unpack(b.get_data()) { Ok(sb) } else { Err(anyhow!("couldn't unpack superblock")) diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index e9c071c..41990cc 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -89,7 +89,11 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result { let b = engine.read(loc)?; - if let Ok((_, sb)) = unpack(&b.get_data()) { + if metadata_block_type(b.get_data()) != BT::THIN_SUPERBLOCK { + return Err(anyhow!("bad checksum in superblock")); + } + + if let Ok((_, sb)) = unpack(b.get_data()) { Ok(sb) } else { Err(anyhow!("couldn't unpack superblock")) From 0d87619a931d76397e66dbf68e2a4f4bf97ac692 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 27 Aug 2021 17:00:10 +0800 Subject: [PATCH 7/7] [thin_repair/thin_dump (rust)] Show repairing progress --- src/thin/dump.rs | 7 ++++++- src/thin/metadata_repair.rs | 40 ++++++++++++++++++++++++++++++++++--- src/thin/repair.rs | 8 ++++++-- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/thin/dump.rs b/src/thin/dump.rs index 7127c16..2cbb019 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -316,7 +316,12 @@ pub fn dump(opts: ThinDumpOptions) -> Result<()> { let ctx = mk_context(&opts)?; let sb; if opts.repair { - sb = read_or_rebuild_superblock(ctx.engine.clone(), SUPERBLOCK_LOCATION, &opts.overrides)?; + sb = read_or_rebuild_superblock( + ctx.engine.clone(), + ctx.report.clone(), + SUPERBLOCK_LOCATION, + &opts.overrides, + )?; } else { sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; } diff --git a/src/thin/metadata_repair.rs b/src/thin/metadata_repair.rs index 43f095c..d96bafe 100644 --- a/src/thin/metadata_repair.rs +++ b/src/thin/metadata_repair.rs @@ -11,6 +11,7 @@ use crate::pdata::btree::*; use crate::pdata::btree_walker::*; use crate::pdata::space_map_common::*; use crate::pdata::unpack::Unpack; +use crate::report::Report; use crate::thin::block_time::*; use crate::thin::device_detail::*; use crate::thin::superblock::*; @@ -188,10 +189,11 @@ struct NodeCollector { examined: FixedBitSet, referenced: FixedBitSet, infos: BTreeMap, + report: Arc, } impl NodeCollector { - pub fn new(engine: Arc) -> NodeCollector { + pub fn new(engine: Arc, report: Arc) -> NodeCollector { let nr_blocks = engine.get_nr_blocks(); NodeCollector { engine, @@ -199,6 +201,7 @@ impl NodeCollector { examined: FixedBitSet::with_capacity(nr_blocks as usize), referenced: FixedBitSet::with_capacity(nr_blocks as usize), infos: BTreeMap::::new(), + report, } } @@ -663,10 +666,39 @@ impl NodeCollector { }) } + fn log_results(&self, dev_roots: &[u64], details_roots: &[u64], pairs: &[(u64, u64)]) { + self.report + .info(&format!("mapping candidates ({}):", dev_roots.len())); + for dev_root in dev_roots { + if let Ok(NodeInfo::Dev(info)) = self.read_info(*dev_root) { + self.report.info(&format!("b={}, nr_devices={}, nr_mappings={}, highest_mapped={}, age={}, time_counts={:?}", + info.b, info.nr_devices, info.nr_mappings, info.highest_mapped_data_block, info.age, info.time_counts)); + } + } + + self.report + .info(&format!("\ndevice candidates ({}):", details_roots.len())); + for details_root in details_roots { + if let Ok(NodeInfo::Details(info)) = self.read_info(*details_root) { + self.report.info(&format!( + "b={}, nr_devices={}, nr_mappings={}, max_tid={}, age={}", + info._b, info.nr_devices, info.nr_mappings, info.max_tid, info.age + )); + } + } + + self.report + .info(&format!("\ncompatible roots ({}):", pairs.len())); + for pair in pairs { + self.report.info(&format!("({}, {})", pair.0, pair.1)); + } + } + pub fn find_roots(mut self) -> Result { self.collect_infos()?; let (dev_roots, details_roots) = self.gather_roots()?; let pairs = self.find_root_pairs(&dev_roots, &details_roots)?; + self.log_results(&dev_roots, &details_roots, &pairs); if pairs.is_empty() { return Err(anyhow!("no compatible roots found")); @@ -720,6 +752,7 @@ pub fn is_superblock_consistent( pub fn rebuild_superblock( engine: Arc, + report: Arc, ref_sb: Option, opts: &SuperblockOverrides, ) -> Result { @@ -735,7 +768,7 @@ pub fn rebuild_superblock( }) .and_then(check_data_block_size)?; - let c = NodeCollector::new(engine.clone()); + let c = NodeCollector::new(engine.clone(), report); let roots = c.find_roots()?; let transaction_id = opts @@ -782,6 +815,7 @@ pub fn rebuild_superblock( pub fn read_or_rebuild_superblock( engine: Arc, + report: Arc, loc: u64, opts: &SuperblockOverrides, ) -> Result { @@ -791,7 +825,7 @@ pub fn read_or_rebuild_superblock( let ref_sb = e .downcast_ref::() .and_then(|err| err.failed_sb.clone()); - rebuild_superblock(engine, ref_sb, opts) + rebuild_superblock(engine, report, ref_sb, opts) }) } diff --git a/src/thin/repair.rs b/src/thin/repair.rs index dd5b996..c7ba02b 100644 --- a/src/thin/repair.rs +++ b/src/thin/repair.rs @@ -55,8 +55,12 @@ fn new_context(opts: &ThinRepairOptions) -> Result { pub fn repair(opts: ThinRepairOptions) -> Result<()> { let ctx = new_context(&opts)?; - let sb = - read_or_rebuild_superblock(ctx.engine_in.clone(), SUPERBLOCK_LOCATION, &opts.overrides)?; + let sb = read_or_rebuild_superblock( + ctx.engine_in.clone(), + ctx.report.clone(), + SUPERBLOCK_LOCATION, + &opts.overrides, + )?; let md = build_metadata(ctx.engine_in.clone(), &sb)?; let md = optimise_metadata(md)?;