[thin_check (rust)] Walk mapping tree.

Sync IO, no checks beyond checksumming.
2020-07-28 10:51:48 +01:00 · 2020-07-28 10:51:48 +01:00 · 062a1b8a2c
parent 1398cf31d1
commit 062a1b8a2c
5 changed files with 284 additions and 53 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -26,9 +26,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.31"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f"
+checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b"

 [[package]]
 name = "arrayvec"
@ -361,18 +361,18 @@ checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"

 [[package]]
 name = "pin-project"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17"
+checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa"
 dependencies = [
 "pin-project-internal",
 ]

 [[package]]
 name = "pin-project-internal"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7"
+checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f"
 dependencies = [
 "proc-macro2",
 "quote",
@ -393,9 +393,9 @@ checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea"

 [[package]]
 name = "proc-macro-hack"
-version = "0.5.16"
+version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
+checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598"

 [[package]]
 name = "proc-macro-nested"
@ -562,9 +562,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

 [[package]]
 name = "syn"
-version = "1.0.35"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0"
+checksum = "4cdb98bcb1f9d81d07b536179c269ea15999b5d14ea958196413869445bb5250"
 dependencies = [
 "proc-macro2",
 "quote",
--- a/src/block_manager.rs
+++ b/src/block_manager.rs
@ -29,8 +29,8 @@ impl Block {
        Block { loc, data: ptr }
    }

-    fn get_data(&self) -> &mut [u8] {
-        unsafe { std::slice::from_raw_parts_mut::<'static>(self.data, BLOCK_SIZE) }
+    pub fn get_data<'a>(&self) -> &'a mut [u8] {
+        unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, BLOCK_SIZE) }
    }
 }

@ -47,7 +47,8 @@ impl Drop for Block {

 pub trait IoEngine {
    fn get_nr_blocks(&self) -> u64;
-    fn read(&mut self, blocks: &mut Vec<Block>) -> Result<()>;
+    fn read(&mut self, block: &mut Block) -> Result<()>;
+    fn read_many(&mut self, blocks: &mut Vec<Block>) -> Result<()>;
 }

 fn get_nr_blocks(path: &Path) -> io::Result<u64> {
@ -70,8 +71,6 @@ impl SyncIoEngine {
            .custom_flags(libc::O_DIRECT)
            .open(path)?;

-        let ring = rio::new()?;
-
        Ok(SyncIoEngine {
            nr_blocks: get_nr_blocks(path)?,
            input,
@ -84,10 +83,16 @@ impl IoEngine for SyncIoEngine {
        self.nr_blocks
    }

-    fn read(&mut self, blocks: &mut Vec<Block>) -> Result<()> {
-        for b in blocks.into_iter() {
-            self.input.seek(io::SeekFrom::Start(0))?;
-            self.input.read_exact(&mut b.get_data())?;
+    fn read(&mut self, b: &mut Block) -> Result<()> {
+        self.input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?;
+        self.input.read_exact(&mut b.get_data())?;
+
+        Ok(())
+    }
+
+    fn read_many(&mut self, blocks: &mut Vec<Block>) -> Result<()> {
+        for b in blocks {
+            self.read(b);
        }

        Ok(())
--- a/src/checksum.rs
+++ b/src/checksum.rs
@ -14,7 +14,7 @@ fn checksum(buf: &[u8]) -> u32 {
    crc32c(&buf[4..]) ^ 0xffffffff
 }

-#[derive(PartialEq)]
+#[derive(Debug, PartialEq)]
 pub enum BT {
    SUPERBLOCK,
    NODE,
--- a/src/thin/check.rs
+++ b/src/thin/check.rs
@ -1,23 +1,211 @@
+use anyhow::{anyhow, Result};
+use nom::{bytes::complete::*, number::complete::*, IResult};
+use std::collections::HashSet;
 use std::error::Error;
 use std::path::Path;
-use std::time::{Duration, Instant};
-use std::thread;
 use std::sync::{Arc, Mutex};
+use std::thread;
+use std::time::{Duration, Instant};

 use crate::block_manager::{Block, IoEngine, SyncIoEngine, BLOCK_SIZE};
+use crate::checksum;
+use crate::thin::superblock::*;

-pub fn check(dev: &Path) -> Result<(), Box<dyn Error>> {
-    let mut engine = SyncIoEngine::new(dev)?;
-    let count = 4096;
+trait ValueType {
+    type Value;
+    fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>;
+}

-    let mut blocks = Vec::new();
-    for n in 0..count {
-        blocks.push(Block::new(n));
+struct NodeHeader {
+    is_leaf: bool,
+    block: u64,
+    nr_entries: u32,
+    max_entries: u32,
+    value_size: u32,
+}
+
+const INTERNAL_NODE: u32 = 1;
+const LEAF_NODE: u32 = 2;
+
+fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> {
+    let (i, _csum) = le_u32(data)?;
+    let (i, flags) = le_u32(i)?;
+    let (i, block) = le_u64(i)?;
+    let (i, nr_entries) = le_u32(i)?;
+    let (i, max_entries) = le_u32(i)?;
+    let (i, value_size) = le_u32(i)?;
+    let (i, _padding) = le_u32(i)?;
+
+    Ok((
+        i,
+        NodeHeader {
+            is_leaf: flags == LEAF_NODE,
+            block,
+            nr_entries,
+            max_entries,
+            value_size,
+        },
+    ))
+}
+
+enum Node<V: ValueType> {
+    Internal {
+        header: NodeHeader,
+        keys: Vec<u64>,
+        values: Vec<u64>,
+    },
+    Leaf {
+        header: NodeHeader,
+        keys: Vec<u64>,
+        values: Vec<V::Value>,
+    },
+}
+
+fn unpack_node_<V: ValueType>(data: &[u8]) -> IResult<&[u8], Node<V>> {
+    use nom::multi::count;
+
+    let (i, header) = unpack_node_header(data)?;
+    let (i, keys) = count(le_u64, header.nr_entries as usize)(i)?;
+
+    let nr_free = header.max_entries - header.nr_entries;
+    let (i, _padding) = count(le_u64, nr_free as usize)(i)?;
+
+    if header.is_leaf {
+        let (i, values) = count(V::unpack, header.nr_entries as usize)(i)?;
+        Ok((
+            i,
+            Node::Leaf {
+                header,
+                keys,
+                values,
+            },
+        ))
+    } else {
+        let (i, values) = count(le_u64, header.nr_entries as usize)(i)?;
+        Ok((
+            i,
+            Node::Internal {
+                header,
+                keys,
+                values,
+            },
+        ))
+    }
+}
+
+fn unpack_node<V: ValueType>(data: &[u8]) -> Result<Node<V>> {
+    if let Ok((_i, node)) = unpack_node_(data) {
+        Ok(node)
+    } else {
+        Err(anyhow!("couldn't unpack btree node"))
+    }
+}
+
+struct ValueU64;
+
+impl ValueType for ValueU64 {
+    type Value = u64;
+    fn unpack(i: &[u8]) -> IResult<&[u8], u64> {
+        le_u64(i)
+    }
+}
+
+struct BlockTime {
+    block: u64,
+    time: u32,
+}
+
+struct ValueBlockTime;
+
+impl ValueType for ValueBlockTime {
+    type Value = BlockTime;
+    fn unpack(i: &[u8]) -> IResult<&[u8], BlockTime> {
+        let (i, n) = le_u64(i)?;
+        let block = n >> 24;
+        let time = n & ((1 << 24) - 1);
+
+        Ok((
+            i,
+            BlockTime {
+                block,
+                time: time as u32,
+            },
+        ))
+    }
+}
+
+enum MappingLevel {
+    Top,
+    Bottom,
+}
+
+fn walk_mapping_tree<E: IoEngine>(
+    engine: &mut E,
+    seen: &mut HashSet<u64>,
+    level: MappingLevel,
+    b: u64,
+) -> Result<()> {
+    if seen.contains(&b) {
+        return Ok(());
+    } else {
+        seen.insert(b);
    }

-    let now = Instant::now();
-    engine.read(&mut blocks)?;
-    println!("read {} blocks in {} ms", count, now.elapsed().as_millis());
+    let mut b = Block::new(b);
+    engine.read(&mut b)?;
+
+    let bt = checksum::metadata_block_type(b.get_data());
+    if bt != checksum::BT::NODE {
+        return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt));
+    }
+
+    match level {
+        MappingLevel::Top => {
+            let node = unpack_node::<ValueU64>(&b.get_data())?;
+            match node {
+                Node::Leaf {header: header, keys: _keys, values} => {
+                    for b in &values {
+                        walk_mapping_tree(engine, seen, MappingLevel::Bottom, *b)?;
+                    }
+                },
+                Node::Internal {header: header, keys: _keys, values} => {
+                    for b in &values {
+                        walk_mapping_tree(engine, seen, MappingLevel::Top, *b)?;
+                    }
+                },
+            }
+        },
+        MappingLevel::Bottom => {
+            let node = unpack_node::<ValueBlockTime>(&b.get_data())?;
+            match node {
+                Node::Leaf {header: header, keys: _keys, values} => {
+                    // FIXME: check in bounds
+                },
+                Node::Internal {header: header, keys: _keys, values} => {
+                    for b in &values {
+                        walk_mapping_tree(engine, seen, MappingLevel::Bottom, *b)?;
+                    }
+                },
+            }
+        }
+    }
+
+    Ok(())
+}
+
+pub fn check(dev: &Path) -> Result<()> {
+    let mut engine = SyncIoEngine::new(dev)?;
+
+    let now = Instant::now();
+    let sb = read_superblock(&mut engine, SUPERBLOCK_LOCATION)?;
+    eprintln!("{:?}", sb);
+    let mut seen = HashSet::new();
+    walk_mapping_tree(&mut engine, &mut seen, MappingLevel::Top, sb.mapping_root)?;
+    println!(
+        "read superblock, mapping root at {}, {} ms",
+        sb.mapping_root,
+        now.elapsed().as_millis()
+    );

    Ok(())
 }
--- a/src/thin/superblock.rs
+++ b/src/thin/superblock.rs
@ -1,20 +1,26 @@
 use crate::block_manager::*;
+use crate::block_manager::*;
 use crate::checksum::*;
+use anyhow::{anyhow, Result};
+use nom::{bytes::complete::*, number::complete::*, IResult};

+pub const SUPERBLOCK_LOCATION: u64 = 0;
+const UUID_SIZE: usize = 16;
 const SPACE_MAP_ROOT_SIZE: usize = 128;

+#[derive(Debug)]
 pub struct Superblock {
-    block: u64,
-    uuid: String,
-    version: u32,
-    time: u32,
-    transaction_id: u64,
-    metadata_snap: u64,
-    data_sm_root: [u8; SPACE_MAP_ROOT_SIZE],
-    metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE],
-    mapping_root: u64,
-    details_root: u64,
-    data_block_size: u32,
+    pub block: u64,
+    //uuid: [u8; UUID_SIZE],
+    pub version: u32,
+    pub time: u32,
+    pub transaction_id: u64,
+    pub metadata_snap: u64,
+    //data_sm_root: [u8; SPACE_MAP_ROOT_SIZE],
+    //metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE],
+    pub mapping_root: u64,
+    pub details_root: u64,
+    pub data_block_size: u32,
 }

 pub enum CheckSeverity {
@ -43,19 +49,51 @@ struct SuperblockError {
    kind: ErrorType,
 }

-/*
-use SuperblockDamage::*;
+fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
+    let (i, _csum) = le_u32(data)?;
+    let (i, _flags) = le_u32(i)?;
+    let (i, block) = le_u64(i)?;
+    let (i, _uuid) = take(16usize)(i)?;
+    let (i, _magic) = le_u64(i)?;
+    let (i, version) = le_u32(i)?;
+    let (i, time) = le_u32(i)?;
+    let (i, transaction_id) = le_u64(i)?;
+    let (i, metadata_snap) = le_u64(i)?;
+    let (i, _data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
+    let (i, _metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
+    let (i, mapping_root) = le_u64(i)?;
+    let (i, details_root) = le_u64(i)?;
+    let (i, data_block_size) = le_u32(i)?;
+    let (i, _metadata_block_size) = le_u32(i)?;
+    let (i, _metadata_nr_blocks) = le_u64(i)?;

-//------------------------------
+    Ok((
+        i,
+        Superblock {
+            block,
+            //uuid: uuid[0..UUID_SIZE],
+            version,
+            time,
+            transaction_id,
+            metadata_snap,
+            //data_sm_root,
+            //metadata_sm_root,
+            mapping_root,
+            details_root,
+            data_block_size,
+        },
+    ))
+}

-pub fn check_type(b: &Block) -> Result<()> {
-    match metadata_block_type(&b.data[0..]) {
-        SUPERBLOCK => Ok(()),
-        NODE => Err(Box::new(BadBlockType("BTree Node"))),
-        INDEX => Err(Box::new(BadBlockType("Space Map Index"))),
-        BITMAP => Err(Box::new(BadBlockType("Space Map Bitmap"))),
-        UNKNOWN => Err(Box::new(BadChecksum)),
+pub fn read_superblock<E: IoEngine>(engine: &mut E, loc: u64) -> Result<Superblock> {
+    let mut b = Block::new(loc);
+    engine.read(&mut b)?;
+
+    if let Ok((_, sb)) = unpack(&b.get_data()) {
+        Ok(sb)
+    } else {
+        Err(anyhow!("couldn't unpack superblock"))
    }
 }
-*/
+
 //------------------------------