[thin_check (rust)] start decoding the space maps.

src/block_manager.rs

@@ -18,7 +18,7 @@ const ALIGN: usize = 4096;
 #[derive(Debug)]
 pub struct Block {
     pub loc: u64,
-    data: *mut u8,
+    pub data: *mut u8,
 }
 
 impl Block {

src/pdata/btree.rs

@@ -18,6 +18,15 @@ pub trait Unpack {
         Self: std::marker::Sized;
 }
 
+pub fn unpack<U: Unpack>(data: &[u8]) -> Result<U> {
+    match U::unpack(data) {
+        Err(_e) => {
+            Err(anyhow!("couldn't parse SMRoot"))
+        },
+        Ok((_i, v)) => Ok(v),
+    }
+}
+
 const NODE_HEADER_SIZE: usize = 32;
 
 pub struct NodeHeader {
@@ -164,6 +173,16 @@ impl Unpack for u64 {
     }
 }
 
+impl Unpack for u32 {
+    fn disk_size() -> u32 {
+        4
+    }
+
+    fn unpack(i: &[u8]) -> IResult<&[u8], u32> {
+        le_u32(i)
+    }
+}
+
 //------------------------------------------
 
 pub trait NodeVisitor<V: Unpack> {

src/pdata/mod.rs

@@ -1,2 +1,3 @@
 pub mod btree;
+pub mod space_map;
 

src/pdata/space_map.rs

@@ -0,0 +1,135 @@
+use anyhow::{anyhow, Result};
+use nom::{number::complete::*, IResult};
+
+use crate::block_manager::*;
+use crate::pdata::btree::Unpack;
+
+//------------------------------------------
+
+#[derive(Debug)]
+pub struct SMRoot {
+    pub nr_blocks: u64,
+    pub nr_allocated: u64,
+    pub bitmap_root: u64,
+    pub ref_count_root: u64,
+}
+
+pub fn unpack_root(data: &[u8]) -> Result<SMRoot> {
+    match SMRoot::unpack(data) {
+        Err(_e) => {
+            Err(anyhow!("couldn't parse SMRoot"))
+        },
+        Ok((_i, v)) => Ok(v),
+    }
+}
+
+impl Unpack for SMRoot {
+    fn disk_size() -> u32 {
+        32
+    }
+    
+    fn unpack(data: &[u8]) -> IResult<&[u8], SMRoot> {
+        let (i, nr_blocks) = le_u64(data)?;
+        let (i, nr_allocated) = le_u64(i)?;
+        let (i, bitmap_root) = le_u64(i)?;
+        let (i, ref_count_root) = le_u64(i)?;
+
+        Ok  ((i, SMRoot {
+            nr_blocks,
+            nr_allocated,
+            bitmap_root,
+            ref_count_root,
+        }))
+    }
+}
+
+//------------------------------------------
+
+#[derive(Clone, Debug)]
+pub struct IndexEntry {
+    pub blocknr: u64,
+    pub nr_free: u32,
+    pub none_free_before: u32,
+}
+
+impl Unpack for IndexEntry {
+    fn disk_size() -> u32 {
+        16
+    }
+
+    fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
+        let (i, blocknr) = le_u64(data)?;
+        let (i, nr_free) = le_u32(i)?;
+        let (i, none_free_before) = le_u32(i)?;
+
+        Ok((i, IndexEntry {blocknr, nr_free, none_free_before}))
+    }
+}
+
+//------------------------------------------
+
+#[derive(Debug)]
+pub struct BitmapHeader {
+    pub csum: u32,
+    pub not_used: u32,
+    pub blocknr: u64,
+}
+
+impl Unpack for BitmapHeader {
+    fn disk_size() -> u32 {
+        16
+    }
+
+    fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
+        let (i, csum) = le_u32(data)?;
+        let (i, not_used) = le_u32(i)?;
+        let (i, blocknr) = le_u64(i)?;
+
+        Ok((i, BitmapHeader {csum, not_used, blocknr} ))
+    }
+}
+
+#[derive(Debug)]
+pub enum BitmapEntry {
+    Small(u8),
+    Overflow,
+}
+
+#[derive(Debug)]
+pub struct Bitmap {
+    pub header: BitmapHeader,
+    pub entries: Vec<BitmapEntry>,
+}
+
+impl Unpack for Bitmap {
+    fn disk_size() -> u32 {
+        BLOCK_SIZE as u32
+    }
+
+    fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
+        let (mut i, header) = BitmapHeader::unpack(data)?;
+
+	let mut entries = Vec::new();
+        let nr_words = (BLOCK_SIZE - BitmapHeader::disk_size() as usize) / 8;
+        for _w in 0..nr_words {
+            let (tmp, mut word) = le_u64(i)?;
+
+            for _b in 0..32 {
+                let val = word & 0x3;
+                word = word >> 2;
+
+                if val < 3 {
+                    entries.push(BitmapEntry::Small(val as u8));
+                } else {
+                    entries.push(BitmapEntry::Overflow);
+                }
+            }
+
+            i = tmp;
+        }
+
+        Ok((i, Bitmap {header, entries}))
+    }
+}
+
+//------------------------------------------

src/shrink/toplevel.rs

@@ -133,6 +133,7 @@ impl<W: Write> xml::MetadataVisitor for Pass2<W> {
 }
 
 //---------------------------------------
+
 type BlockRange = std::ops::Range<u64>;
 
 fn bits_to_ranges(bits: &FixedBitSet) -> Vec<BlockRange> {

src/thin/check.rs

@@ -8,8 +8,10 @@ use std::time::Instant;
 use threadpool::ThreadPool;
 
 use crate::block_manager::{AsyncIoEngine, Block, IoEngine};
-use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack};
+use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack, unpack};
+use crate::pdata::space_map::*;
 use crate::thin::superblock::*;
+use crate::checksum;
 
 //------------------------------------------
 
@@ -128,7 +130,64 @@ impl NodeVisitor<DeviceDetail> for DeviceVisitor {
             for n in 0..keys.len() {
                 let k = keys[n] as u32;
                 let v = values[n].clone();
-                self.devs.insert(k, v.clone());
+                self.devs.insert(k, v);
+            }
+        }
+
+        Ok(())
+    }
+}
+
+//------------------------------------------
+
+struct IndexVisitor {
+    entries: Vec<IndexEntry>,
+}
+
+impl NodeVisitor<IndexEntry> for IndexVisitor {
+    fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node<IndexEntry>) -> Result<()> {
+        if let Node::Leaf {
+            header: _h,
+            keys,
+            values,
+        } = node {
+            for n in 0..keys.len() {
+                // FIXME: check keys are in incremental order
+                let v = values[n].clone();
+                self.entries.push(v);
+            }
+        }
+
+        Ok(())
+    }
+}
+
+//------------------------------------------
+
+// FIXME: move to btree
+struct ValueCollector<V> {
+    values: Vec<(u64, V)>,
+}
+
+impl<V> ValueCollector<V> {
+    fn new() -> ValueCollector<V> {
+        ValueCollector {
+            values: Vec::new(),
+        }
+    }
+}
+
+impl<V: Unpack + Clone> NodeVisitor<V> for ValueCollector<V> {
+    fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node<V>) -> Result<()> {
+        if let Node::Leaf {
+            header: _h,
+            keys,
+            values,
+        } = node {
+            for n in 0..keys.len() {
+                let k = keys[n];
+                let v = values[n].clone();
+                self.values.push((k, v));
             }
         }
 
@@ -145,6 +204,7 @@ pub fn check(dev: &Path) -> Result<()> {
     let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?;
     eprintln!("{:?}", sb);
 
+    // device details
     {
         let mut visitor = DeviceVisitor::new();
         let mut w = BTreeWalker::new(engine.clone(), false);
@@ -152,6 +212,8 @@ pub fn check(dev: &Path) -> Result<()> {
         println!("found {} devices", visitor.devs.len());
     }
 
+/*
+    // mapping top level
     let mut roots = HashMap::new();
     {
         let mut visitor = TopLevelVisitor { roots: &mut roots };
@@ -160,8 +222,9 @@ pub fn check(dev: &Path) -> Result<()> {
         println!("read mapping tree in {} ms", now.elapsed().as_millis());
     }
 
-    // FIXME: with a thread pool we need to return errors another way.
+    // mapping bottom level
     {
+        // FIXME: with a thread pool we need to return errors another way.
         let nr_workers = 4;
         let pool = ThreadPool::new(nr_workers);
         let seen = Arc::new(Mutex::new(FixedBitSet::with_capacity(
@@ -179,6 +242,43 @@ pub fn check(dev: &Path) -> Result<()> {
 
         pool.join();
     }
+    */
+
+    // data space map
+    {
+        let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
+        eprintln!("data root: {:?}", root);
+
+        // overflow btree
+        let mut overflow: HashMap<u64, u32> = HashMap::new();
+        {
+            let mut v: ValueCollector<u32> = ValueCollector::new();
+            let mut w = BTreeWalker::new(engine.clone(), false);
+            w.walk(&mut v, root.ref_count_root)?;
+
+            for (k, v) in v.values {
+                overflow.insert(k, v);
+            }
+        }
+        eprintln!("{} overflow entries", overflow.len());
+
+        // Bitmaps
+        let mut v = IndexVisitor {entries: Vec::new()};
+        let mut w = BTreeWalker::new(engine.clone(), false);
+        let _result = w.walk(&mut v, root.bitmap_root);
+        eprintln!("{} index entries", v.entries.len());
+
+        for i in v.entries {
+            let mut b = Block::new(i.blocknr);
+            engine.read(&mut b)?;
+            
+            if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP {
+                return Err(anyhow!("Index entry points to block ({}) that isn't a bitmap", b.loc));
+            }
+
+            let bitmap = unpack::<Bitmap>(b.get_data())?;
+        }
+    }
 
     Ok(())
 }

src/thin/superblock.rs

@@ -14,8 +14,8 @@ pub struct Superblock {
     pub time: u32,
     pub transaction_id: u64,
     pub metadata_snap: u64,
-    //data_sm_root: [u8; SPACE_MAP_ROOT_SIZE],
-    //metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE],
+    pub data_sm_root: Vec<u8>,
+    pub metadata_sm_root: Vec<u8>,
     pub mapping_root: u64,
     pub details_root: u64,
     pub data_block_size: u32,
@@ -59,8 +59,8 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
     let (i, time) = le_u32(i)?;
     let (i, transaction_id) = le_u64(i)?;
     let (i, metadata_snap) = le_u64(i)?;
-    let (i, _data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
-    let (i, _metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
+    let (i, data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
+    let (i, metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
     let (i, mapping_root) = le_u64(i)?;
     let (i, details_root) = le_u64(i)?;
     let (i, data_block_size) = le_u32(i)?;
@@ -76,8 +76,8 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
             time,
             transaction_id,
             metadata_snap,
-            //data_sm_root,
-            //metadata_sm_root,
+            data_sm_root: data_sm_root.to_vec(),
+            metadata_sm_root: metadata_sm_root.to_vec(),
             mapping_root,
             details_root,
             data_block_size,