From 4a0582bb5db214620a369ecf3b0cac65f3e6a755 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 6 Aug 2020 07:51:48 +0100 Subject: [PATCH] [thin_check (rust)] start decoding the space maps. --- src/block_manager.rs | 2 +- src/pdata/btree.rs | 19 ++++++ src/pdata/mod.rs | 1 + src/pdata/space_map.rs | 135 +++++++++++++++++++++++++++++++++++++++++ src/shrink/toplevel.rs | 1 + src/thin/check.rs | 106 +++++++++++++++++++++++++++++++- src/thin/superblock.rs | 12 ++-- 7 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 src/pdata/space_map.rs diff --git a/src/block_manager.rs b/src/block_manager.rs index 7aebbaf..1d5e061 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -18,7 +18,7 @@ const ALIGN: usize = 4096; #[derive(Debug)] pub struct Block { pub loc: u64, - data: *mut u8, + pub data: *mut u8, } impl Block { diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 00c8620..81802df 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -18,6 +18,15 @@ pub trait Unpack { Self: std::marker::Sized; } +pub fn unpack(data: &[u8]) -> Result { + match U::unpack(data) { + Err(_e) => { + Err(anyhow!("couldn't parse SMRoot")) + }, + Ok((_i, v)) => Ok(v), + } +} + const NODE_HEADER_SIZE: usize = 32; pub struct NodeHeader { @@ -164,6 +173,16 @@ impl Unpack for u64 { } } +impl Unpack for u32 { + fn disk_size() -> u32 { + 4 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], u32> { + le_u32(i) + } +} + //------------------------------------------ pub trait NodeVisitor { diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs index 83204ba..3e36813 100644 --- a/src/pdata/mod.rs +++ b/src/pdata/mod.rs @@ -1,2 +1,3 @@ pub mod btree; +pub mod space_map; diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs new file mode 100644 index 0000000..649964f --- /dev/null +++ b/src/pdata/space_map.rs @@ -0,0 +1,135 @@ +use anyhow::{anyhow, Result}; +use nom::{number::complete::*, IResult}; + +use crate::block_manager::*; +use crate::pdata::btree::Unpack; + +//------------------------------------------ + +#[derive(Debug)] +pub struct SMRoot { + pub nr_blocks: u64, + pub nr_allocated: u64, + pub bitmap_root: u64, + pub ref_count_root: u64, +} + +pub fn unpack_root(data: &[u8]) -> Result { + match SMRoot::unpack(data) { + Err(_e) => { + Err(anyhow!("couldn't parse SMRoot")) + }, + Ok((_i, v)) => Ok(v), + } +} + +impl Unpack for SMRoot { + fn disk_size() -> u32 { + 32 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], SMRoot> { + let (i, nr_blocks) = le_u64(data)?; + let (i, nr_allocated) = le_u64(i)?; + let (i, bitmap_root) = le_u64(i)?; + let (i, ref_count_root) = le_u64(i)?; + + Ok ((i, SMRoot { + nr_blocks, + nr_allocated, + bitmap_root, + ref_count_root, + })) + } +} + +//------------------------------------------ + +#[derive(Clone, Debug)] +pub struct IndexEntry { + pub blocknr: u64, + pub nr_free: u32, + pub none_free_before: u32, +} + +impl Unpack for IndexEntry { + fn disk_size() -> u32 { + 16 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (i, blocknr) = le_u64(data)?; + let (i, nr_free) = le_u32(i)?; + let (i, none_free_before) = le_u32(i)?; + + Ok((i, IndexEntry {blocknr, nr_free, none_free_before})) + } +} + +//------------------------------------------ + +#[derive(Debug)] +pub struct BitmapHeader { + pub csum: u32, + pub not_used: u32, + pub blocknr: u64, +} + +impl Unpack for BitmapHeader { + fn disk_size() -> u32 { + 16 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (i, csum) = le_u32(data)?; + let (i, not_used) = le_u32(i)?; + let (i, blocknr) = le_u64(i)?; + + Ok((i, BitmapHeader {csum, not_used, blocknr} )) + } +} + +#[derive(Debug)] +pub enum BitmapEntry { + Small(u8), + Overflow, +} + +#[derive(Debug)] +pub struct Bitmap { + pub header: BitmapHeader, + pub entries: Vec, +} + +impl Unpack for Bitmap { + fn disk_size() -> u32 { + BLOCK_SIZE as u32 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (mut i, header) = BitmapHeader::unpack(data)?; + + let mut entries = Vec::new(); + let nr_words = (BLOCK_SIZE - BitmapHeader::disk_size() as usize) / 8; + for _w in 0..nr_words { + let (tmp, mut word) = le_u64(i)?; + + for _b in 0..32 { + let val = word & 0x3; + word = word >> 2; + + if val < 3 { + entries.push(BitmapEntry::Small(val as u8)); + } else { + entries.push(BitmapEntry::Overflow); + } + } + + i = tmp; + } + + Ok((i, Bitmap {header, entries})) + } +} + +//------------------------------------------ diff --git a/src/shrink/toplevel.rs b/src/shrink/toplevel.rs index 38ffe82..1a3d8a9 100644 --- a/src/shrink/toplevel.rs +++ b/src/shrink/toplevel.rs @@ -133,6 +133,7 @@ impl xml::MetadataVisitor for Pass2 { } //--------------------------------------- + type BlockRange = std::ops::Range; fn bits_to_ranges(bits: &FixedBitSet) -> Vec { diff --git a/src/thin/check.rs b/src/thin/check.rs index 3611248..6461ad8 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -8,8 +8,10 @@ use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; -use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack}; +use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack, unpack}; +use crate::pdata::space_map::*; use crate::thin::superblock::*; +use crate::checksum; //------------------------------------------ @@ -128,7 +130,7 @@ impl NodeVisitor for DeviceVisitor { for n in 0..keys.len() { let k = keys[n] as u32; let v = values[n].clone(); - self.devs.insert(k, v.clone()); + self.devs.insert(k, v); } } @@ -138,6 +140,63 @@ impl NodeVisitor for DeviceVisitor { //------------------------------------------ +struct IndexVisitor { + entries: Vec, +} + +impl NodeVisitor for IndexVisitor { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node { + for n in 0..keys.len() { + // FIXME: check keys are in incremental order + let v = values[n].clone(); + self.entries.push(v); + } + } + + Ok(()) + } +} + +//------------------------------------------ + +// FIXME: move to btree +struct ValueCollector { + values: Vec<(u64, V)>, +} + +impl ValueCollector { + fn new() -> ValueCollector { + ValueCollector { + values: Vec::new(), + } + } +} + +impl NodeVisitor for ValueCollector { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node { + for n in 0..keys.len() { + let k = keys[n]; + let v = values[n].clone(); + self.values.push((k, v)); + } + } + + Ok(()) + } +} + +//------------------------------------------ + pub fn check(dev: &Path) -> Result<()> { let engine = Arc::new(AsyncIoEngine::new(dev, 256)?); @@ -145,6 +204,7 @@ pub fn check(dev: &Path) -> Result<()> { let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; eprintln!("{:?}", sb); + // device details { let mut visitor = DeviceVisitor::new(); let mut w = BTreeWalker::new(engine.clone(), false); @@ -152,6 +212,8 @@ pub fn check(dev: &Path) -> Result<()> { println!("found {} devices", visitor.devs.len()); } +/* + // mapping top level let mut roots = HashMap::new(); { let mut visitor = TopLevelVisitor { roots: &mut roots }; @@ -160,8 +222,9 @@ pub fn check(dev: &Path) -> Result<()> { println!("read mapping tree in {} ms", now.elapsed().as_millis()); } - // FIXME: with a thread pool we need to return errors another way. + // mapping bottom level { + // FIXME: with a thread pool we need to return errors another way. let nr_workers = 4; let pool = ThreadPool::new(nr_workers); let seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( @@ -179,6 +242,43 @@ pub fn check(dev: &Path) -> Result<()> { pool.join(); } + */ + + // data space map + { + let root = unpack::(&sb.data_sm_root[0..])?; + eprintln!("data root: {:?}", root); + + // overflow btree + let mut overflow: HashMap = HashMap::new(); + { + let mut v: ValueCollector = ValueCollector::new(); + let mut w = BTreeWalker::new(engine.clone(), false); + w.walk(&mut v, root.ref_count_root)?; + + for (k, v) in v.values { + overflow.insert(k, v); + } + } + eprintln!("{} overflow entries", overflow.len()); + + // Bitmaps + let mut v = IndexVisitor {entries: Vec::new()}; + let mut w = BTreeWalker::new(engine.clone(), false); + let _result = w.walk(&mut v, root.bitmap_root); + eprintln!("{} index entries", v.entries.len()); + + for i in v.entries { + let mut b = Block::new(i.blocknr); + engine.read(&mut b)?; + + if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { + return Err(anyhow!("Index entry points to block ({}) that isn't a bitmap", b.loc)); + } + + let bitmap = unpack::(b.get_data())?; + } + } Ok(()) } diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index 5001eb6..8c1e738 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -14,8 +14,8 @@ pub struct Superblock { pub time: u32, pub transaction_id: u64, pub metadata_snap: u64, - //data_sm_root: [u8; SPACE_MAP_ROOT_SIZE], - //metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE], + pub data_sm_root: Vec, + pub metadata_sm_root: Vec, pub mapping_root: u64, pub details_root: u64, pub data_block_size: u32, @@ -59,8 +59,8 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { let (i, time) = le_u32(i)?; let (i, transaction_id) = le_u64(i)?; let (i, metadata_snap) = le_u64(i)?; - let (i, _data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; - let (i, _metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; let (i, mapping_root) = le_u64(i)?; let (i, details_root) = le_u64(i)?; let (i, data_block_size) = le_u32(i)?; @@ -76,8 +76,8 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { time, transaction_id, metadata_snap, - //data_sm_root, - //metadata_sm_root, + data_sm_root: data_sm_root.to_vec(), + metadata_sm_root: metadata_sm_root.to_vec(), mapping_root, details_root, data_block_size,