From 8493cf708183ed829d159afa9b909ba5d576a5a3 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 16 Sep 2020 15:10:01 +0100 Subject: [PATCH] [thin_explore] First code drop --- Cargo.lock | 62 +++- Cargo.toml | 2 + src/bin/thin_explore.rs | 555 +++++++++++++++++++++++++++++++++++ src/pdata/btree.rs | 629 ++++++++++++++++++++++++++++++++++------ src/thin/block_time.rs | 40 +++ src/thin/check.rs | 24 +- src/thin/mod.rs | 1 + src/thin/superblock.rs | 20 +- 8 files changed, 1219 insertions(+), 114 deletions(-) create mode 100644 src/bin/thin_explore.rs create mode 100644 src/thin/block_time.rs diff --git a/Cargo.lock b/Cargo.lock index bd80e7c..904d37b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + [[package]] name = "cc" version = "1.0.59" @@ -426,6 +432,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +[[package]] +name = "numtoa" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" + [[package]] name = "once_cell" version = "1.4.1" @@ -488,9 +500,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" +checksum = "175c513d55719db99da20232b06cda8bab6b83ec2d04e3283edf0213c37c1a29" dependencies = [ "unicode-xid", ] @@ -583,6 +595,15 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +dependencies = [ + "redox_syscall", +] + [[package]] name = "regex" version = "1.3.9" @@ -646,9 +667,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" +checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" dependencies = [ "proc-macro2", "quote", @@ -679,6 +700,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "termion" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c22cec9d8978d906be5ac94bceb5a010d885c626c4c8855721a4dbd20e3ac905" +dependencies = [ + "libc", + "numtoa", + "redox_syscall", + "redox_termios", +] + [[package]] name = "termios" version = "0.3.2" @@ -725,8 +758,10 @@ dependencies = [ "quickcheck_macros", "rand", "tempfile", + "termion", "thiserror", "threadpool", + "tui", ] [[package]] @@ -767,6 +802,25 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tui" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a977b0bb2e2033a6fef950f218f13622c3c34e59754b704ce3492dedab1dfe95" +dependencies = [ + "bitflags", + "cassowary", + "termion", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + [[package]] name = "unicode-width" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 0ecfe46..a86f0d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,8 @@ rand = "0.7" tempfile = "3.1" threadpool = "1.8" thiserror = "1.0" +tui = "0.10" +termion = "1.5" [dev-dependencies] json = "0.12" diff --git a/src/bin/thin_explore.rs b/src/bin/thin_explore.rs new file mode 100644 index 0000000..d57decd --- /dev/null +++ b/src/bin/thin_explore.rs @@ -0,0 +1,555 @@ +extern crate clap; + +use anyhow::{anyhow, Result}; +use clap::{App, Arg}; +use std::fmt; +use std::io::{self, Read, Write}; +use std::path::Path; +use std::sync::mpsc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::thread; +use std::time::Duration; + +use termion::clear; +use termion::color; +use termion::event::Key; +use termion::input::TermRead; +use termion::raw::IntoRawMode; + +use tui::{ + backend::{Backend, TermionBackend}, + buffer::Buffer, + layout::{Constraint, Direction, Layout, Rect}, + style::{Color, Modifier, Style}, + terminal::Frame, + text::{Span, Spans}, + widgets::{Block, Borders, List, ListItem, ListState, Row, StatefulWidget, Table, Widget}, + Terminal, +}; + +use thinp::io_engine::*; +use thinp::pdata::btree; +use thinp::pdata::unpack::*; +use thinp::thin::block_time::*; +use thinp::thin::superblock::*; + +//------------------------------------ + +pub enum Event { + Input(I), + Tick, +} + +pub struct Events { + rx: mpsc::Receiver>, + input_handle: thread::JoinHandle<()>, + ignore_exit_key: Arc, + tick_handle: thread::JoinHandle<()>, +} + +#[derive(Debug, Clone, Copy)] +pub struct Config { + pub exit_key: Key, + pub tick_rate: Duration, +} + +impl Default for Config { + fn default() -> Config { + Config { + exit_key: Key::Char('q'), + tick_rate: Duration::from_millis(250), + } + } +} + +impl Events { + pub fn new() -> Events { + Events::with_config(Config::default()) + } + + pub fn with_config(config: Config) -> Events { + let (tx, rx) = mpsc::channel(); + let ignore_exit_key = Arc::new(AtomicBool::new(false)); + let input_handle = { + let tx = tx.clone(); + let ignore_exit_key = ignore_exit_key.clone(); + thread::spawn(move || { + let stdin = io::stdin(); + for evt in stdin.keys() { + if let Ok(key) = evt { + if let Err(err) = tx.send(Event::Input(key)) { + eprintln!("{}", err); + return; + } + if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key { + return; + } + } + } + }) + }; + + let tick_handle = { + thread::spawn(move || loop { + if tx.send(Event::Tick).is_err() { + break; + } + thread::sleep(config.tick_rate); + }) + }; + + Events { + rx, + ignore_exit_key, + input_handle, + tick_handle, + } + } + + pub fn next(&self) -> Result, mpsc::RecvError> { + self.rx.recv() + } + + pub fn disable_exit_key(&mut self) { + self.ignore_exit_key.store(true, Ordering::Relaxed); + } + + pub fn enable_exit_key(&mut self) { + self.ignore_exit_key.store(false, Ordering::Relaxed); + } +} + +//------------------------------------ + +fn ls_next(ls: &mut ListState, max: usize) { + let i = match ls.selected() { + Some(i) => { + if i >= max - 1 { + max - 1 + } else { + i + 1 + } + } + None => 0, + }; + ls.select(Some(i)); +} + +fn ls_previous(ls: &mut ListState, max: usize) { + let i = match ls.selected() { + Some(i) => { + if i == 0 { + 0 + } else { + i - 1 + } + } + None => 0, + }; + ls.select(Some(i)); +} + +//------------------------------------ + +struct SBWidget { + sb: Superblock, +} + +impl Widget for SBWidget { + fn render(self, area: Rect, buf: &mut Buffer) { + let sb = &self.sb; + let flags = ["flags".to_string(), format!("{}", sb.flags)]; + let block = ["block".to_string(), format!("{}", sb.block)]; + let uuid = ["uuid".to_string(), format!("-")]; + let version = ["version".to_string(), format!("{}", sb.version)]; + let time = ["time".to_string(), format!("{}", sb.time)]; + let transaction_id = [ + "transaction_id".to_string(), + format!("{}", sb.transaction_id), + ]; + let metadata_snap = [ + "metadata_snap".to_string(), + if sb.metadata_snap == 0 { + "-".to_string() + } else { + format!("{}", sb.metadata_snap) + }, + ]; + let mapping_root = ["mapping root".to_string(), format!("{}", sb.mapping_root)]; + let details_root = ["details root".to_string(), format!("{}", sb.details_root)]; + let data_block_size = [ + "data block size".to_string(), + format!("{}k", sb.data_block_size * 2), + ]; + + let row_style = Style::default().fg(Color::White); + let table = Table::new( + ["Field", "Value"].iter(), + vec![ + Row::Data(flags.iter()), + Row::Data(block.iter()), + Row::Data(uuid.iter()), + Row::Data(version.iter()), + Row::Data(time.iter()), + Row::Data(transaction_id.iter()), + Row::Data(metadata_snap.iter()), + Row::Data(mapping_root.iter()), + Row::Data(details_root.iter()), + Row::Data(data_block_size.iter()), + ] + .into_iter(), + ) + .block( + Block::default() + .borders(Borders::ALL) + .title("Superblock".to_string()), + ) + .header_style(Style::default().fg(Color::Yellow)) + .widths(&[Constraint::Length(20), Constraint::Length(60)]) + .style(Style::default().fg(Color::White)) + .column_spacing(1); + + Widget::render(table, area, buf); + } +} + +//------------------------------------ + +struct HeaderWidget<'a> { + title: String, + hdr: &'a btree::NodeHeader, +} + +impl<'a> Widget for HeaderWidget<'a> { + fn render(self, area: Rect, buf: &mut Buffer) { + let hdr = &self.hdr; + let kind = [ + "type".to_string(), + match hdr.is_leaf { + true => "LEAF".to_string(), + false => "INTERNAL".to_string(), + }, + ]; + let nr_entries = ["nr_entries".to_string(), format!("{}", hdr.nr_entries)]; + let max_entries = ["max_entries".to_string(), format!("{}", hdr.max_entries)]; + let value_size = ["value size".to_string(), format!("{}", hdr.value_size)]; + + let row_style = Style::default().fg(Color::White); + let table = Table::new( + ["Field", "Value"].iter(), + vec![ + Row::Data(kind.iter()), + Row::Data(nr_entries.iter()), + Row::Data(max_entries.iter()), + Row::Data(value_size.iter()), + ] + .into_iter(), + ) + .block(Block::default().borders(Borders::ALL).title(self.title)) + .header_style(Style::default().fg(Color::Yellow)) + .widths(&[Constraint::Length(20), Constraint::Length(60)]) + .style(Style::default().fg(Color::White)) + .column_spacing(1); + + Widget::render(table, area, buf); + } +} + +fn read_node_header(engine: &dyn IoEngine, loc: u64) -> Result { + let b = engine.read(loc)?; + unpack(&b.get_data()).map_err(|_| anyhow!("couldn't unpack btree header")) +} + +fn read_node(engine: &dyn IoEngine, loc: u64) -> Result> { + let b = engine.read(loc)?; + btree::unpack_node(&b.get_data(), true, false) + .map_err(|_| anyhow!("couldn't unpack btree node")) +} + +//------------------------------------ + +struct NodeWidget<'a, V: Unpack> { + title: String, + node: &'a btree::Node, +} + +impl<'a, V: Unpack + fmt::Display> StatefulWidget for NodeWidget<'a, V> { + type State = ListState; + + fn render(self, area: Rect, buf: &mut Buffer, state: &mut ListState) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Min(10), Constraint::Percentage(80)].as_ref()) + .split(area); + + let hdr = HeaderWidget { + title: self.title, + hdr: self.node.get_header(), + }; + hdr.render(chunks[0], buf); + + let mut items: Vec = Vec::new(); + + match self.node { + btree::Node::Internal { keys, values, .. } => { + for (k, v) in keys.iter().zip(values.iter()) { + items.push(ListItem::new(Span::raw(format!("{} -> {}", k, v)))); + } + } + btree::Node::Leaf { keys, values, .. } => { + for (k, v) in keys.iter().zip(values.iter()) { + items.push(ListItem::new(Span::raw(format!("{} -> {}", k, v)))); + } + } + } + + let items = List::new(items) + .block(Block::default().borders(Borders::ALL).title("Entries")) + .highlight_style( + Style::default() + .bg(Color::LightGreen) + .add_modifier(Modifier::BOLD), + ); + + StatefulWidget::render(items, chunks[1], buf, state); + } +} + +//------------------------------------ + +enum Action { + PushTopLevel(u64), + PushBottomLevel(u32, u64), + PopPanel, +} + +type Frame_<'a, 'b> = Frame<'a, TermionBackend>>>; + +trait Panel { + fn render(&mut self, area: Rect, f: &mut Frame_); + fn input(&mut self, k: Key) -> Option; +} + +struct SBPanel { + sb: Superblock, +} + +impl Panel for SBPanel { + fn render(&mut self, area: Rect, f: &mut Frame_) { + // FIXME: get rid of clone + let w = SBWidget { + sb: self.sb.clone(), + }; + f.render_widget(w, area); + } + + fn input(&mut self, k: Key) -> Option { + None + } +} + +struct TopLevelPanel { + node: btree::Node, + nr_entries: usize, + state: ListState, +} + +impl TopLevelPanel { + fn new(node: btree::Node) -> TopLevelPanel { + let nr_entries = node.get_header().nr_entries as usize; + let mut state = ListState::default(); + state.select(Some(0)); + + TopLevelPanel { + node, + nr_entries, + state, + } + } +} + +impl Panel for TopLevelPanel { + fn render(&mut self, area: Rect, f: &mut Frame_) { + let w = NodeWidget { + title: "Top Level".to_string(), + node: &self.node, // FIXME: get rid of clone + }; + + f.render_stateful_widget(w, area, &mut self.state); + } + + fn input(&mut self, k: Key) -> Option { + match k { + Key::Char('j') | Key::Down => { + ls_next(&mut self.state, self.nr_entries); + None + } + Key::Char('k') | Key::Up => { + ls_previous(&mut self.state, self.nr_entries); + None + } + Key::Char('l') | Key::Right => match &self.node { + btree::Node::Internal { values, .. } => { + Some(Action::PushTopLevel(values[self.state.selected().unwrap()])) + } + btree::Node::Leaf { values, keys, .. } => { + let index = self.state.selected().unwrap(); + + Some(Action::PushBottomLevel(keys[index] as u32, values[index])) + } + }, + Key::Char('h') | Key::Left => Some(Action::PopPanel), + _ => None, + } + } +} + +struct BottomLevelPanel { + thin_id: u32, + node: btree::Node, + nr_entries: usize, + state: ListState, +} + +impl BottomLevelPanel { + fn new(thin_id: u32, node: btree::Node) -> BottomLevelPanel { + let nr_entries = node.get_header().nr_entries as usize; + let mut state = ListState::default(); + state.select(Some(0)); + + BottomLevelPanel { + thin_id, + node, + nr_entries, + state, + } + } +} + +impl Panel for BottomLevelPanel { + fn render(&mut self, area: Rect, f: &mut Frame_) { + let w = NodeWidget { + title: format!("Thin dev #{}", self.thin_id), + node: &self.node, + }; + + f.render_stateful_widget(w, area, &mut self.state); + } + + fn input(&mut self, k: Key) -> Option { + match k { + Key::Char('j') | Key::Down => { + ls_next(&mut self.state, self.nr_entries); + None + } + Key::Char('k') | Key::Up => { + ls_previous(&mut self.state, self.nr_entries); + None + } + Key::Char('l') | Key::Right => match &self.node { + btree::Node::Internal { values, .. } => Some(Action::PushBottomLevel( + self.thin_id, + values[self.state.selected().unwrap()], + )), + _ => None, + }, + + Key::Char('h') | Key::Left => Some(Action::PopPanel), + _ => None, + } + } +} + +//------------------------------------ + +fn explore(path: &Path) -> Result<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock().into_raw_mode()?; + write!(stdout, "{}", termion::clear::All); + + let backend = TermionBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + let path = std::path::Path::new("bz1763895/meta.bin"); + let engine = SyncIoEngine::new(&path, 1, false)?; + + let mut panels: Vec> = Vec::new(); + + let sb = read_superblock(&engine, 0)?; + panels.push(Box::new(SBPanel { sb: sb.clone() })); + + let node = read_node::(&engine, sb.mapping_root)?; + panels.push(Box::new(TopLevelPanel::new(node))); + + let events = Events::new(); + + 'main: loop { + let render_panels = |f: &mut Frame_| { + let chunks = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref()) + .split(f.size()); + + let mut base = panels.len(); + if base >= 2 { + base -= 2; + } else { + base = 0; + } + + for i in base..panels.len() { + panels[i].render(chunks[i - base], f); + } + }; + + terminal.draw(render_panels)?; + + let last = panels.len() - 1; + let active_panel = &mut panels[last]; + if let Event::Input(key) = events.next()? { + match key { + Key::Char('q') => break 'main, + _ => match active_panel.input(key) { + Some(Action::PushTopLevel(b)) => { + let node = read_node::(&engine, b)?; + panels.push(Box::new(TopLevelPanel::new(node))); + } + Some(Action::PushBottomLevel(thin_id, b)) => { + let node = read_node::(&engine, b)?; + panels.push(Box::new(BottomLevelPanel::new(thin_id, node))); + } + Some(Action::PopPanel) => { + panels.pop(); + } + _ => {} + }, + } + } + } + + Ok(()) +} + +//------------------------------------ + +fn main() -> Result<()> { + let parser = App::new("thin_explore") + .version(thinp::version::TOOLS_VERSION) + .about("A text user interface for examining thin metadata.") + .arg( + Arg::with_name("INPUT") + .help("Specify the input device to check") + .required(true) + .index(1), + ); + + let matches = parser.get_matches(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + + explore(&input_file) +} + +//------------------------------------ diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index fca0ae3..c33bdc5 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -1,7 +1,8 @@ -use anyhow::{anyhow, Result}; use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; +use std::fmt; use std::sync::{Arc, Mutex}; +use thiserror::Error; use threadpool::ThreadPool; use crate::checksum; @@ -13,39 +14,346 @@ use crate::pdata::unpack::*; //------------------------------------------ +#[derive(Clone, Debug, PartialEq)] +pub struct KeyRange { + start: Option, + end: Option, // This is the one-past-the-end value +} + +impl fmt::Display for KeyRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match (self.start, self.end) { + (None, None) => write!(f, "[..]"), + (None, Some(e)) => write!(f, "[..{}]", e), + (Some(s), None) => write!(f, "[{}..]", s), + (Some(s), Some(e)) => write!(f, "[{}..{}]", s, e), + } + } +} + +impl KeyRange { + // None will be returned if either range would be zero length + fn split(&self, n: u64) -> Option<(KeyRange, KeyRange)> { + match (self.start, self.end) { + (None, None) => Some(( + KeyRange { + start: None, + end: Some(n), + }, + KeyRange { + start: Some(n), + end: None, + }, + )), + (None, Some(e)) => { + if n < e { + Some(( + KeyRange { + start: None, + end: Some(n), + }, + KeyRange { + start: Some(n), + end: Some(e), + }, + )) + } else { + None + } + } + (Some(s), None) => { + if s < n { + Some(( + KeyRange { + start: Some(s), + end: Some(n), + }, + KeyRange { + start: Some(n), + end: None, + }, + )) + } else { + None + } + } + (Some(s), Some(e)) => { + if s < n && n < e { + Some(( + KeyRange { + start: Some(s), + end: Some(n), + }, + KeyRange { + start: Some(n), + end: Some(e), + }, + )) + } else { + None + } + } + } + } +} + +#[test] +fn test_split_range() { + struct Test(Option, Option, u64, Option<(KeyRange, KeyRange)>); + + let tests = vec![ + Test( + None, + None, + 100, + Some(( + KeyRange { + start: None, + end: Some(100), + }, + KeyRange { + start: Some(100), + end: None, + }, + )), + ), + Test(None, Some(100), 1000, None), + Test( + None, + Some(100), + 50, + Some(( + KeyRange { + start: None, + end: Some(50), + }, + KeyRange { + start: Some(50), + end: Some(100), + }, + )), + ), + Test(None, Some(100), 100, None), + Test(Some(100), None, 50, None), + Test( + Some(100), + None, + 150, + Some(( + KeyRange { + start: Some(100), + end: Some(150), + }, + KeyRange { + start: Some(150), + end: None, + }, + )), + ), + Test(Some(100), Some(200), 50, None), + Test(Some(100), Some(200), 250, None), + Test( + Some(100), + Some(200), + 150, + Some(( + KeyRange { + start: Some(100), + end: Some(150), + }, + KeyRange { + start: Some(150), + end: Some(200), + }, + )), + ), + ]; + + for Test(start, end, n, expected) in tests { + let kr = KeyRange { start, end }; + let actual = kr.split(n); + assert_eq!(actual, expected); + } +} + +fn split_one(kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRange)> { + match kr.split(k) { + None => { + return Err(node_err(&format!( + "couldn't split key range {} at {}", + kr, k + ))); + } + Some(pair) => Ok(pair), + } +} + +fn split_key_ranges_(kr: &KeyRange, keys: &[u64]) -> Result> { + let mut krs = Vec::with_capacity(keys.len()); + + if keys.len() == 0 { + return Err(node_err("split_key_ranges: no keys present")); + } + + // The first key gives the lower bound + let mut kr = KeyRange { + start: Some(keys[0]), + end: kr.end, + }; + + for i in 1..keys.len() { + let (first, rest) = split_one(&kr, keys[i])?; + krs.push(first); + kr = rest; + } + + krs.push(kr); + + Ok(krs) +} + +fn split_key_ranges(kr: &KeyRange, keys: &[u64]) -> Result> { + let msg = format!("split: {:?} at {:?}", &kr, &keys); + let r = split_key_ranges_(kr, keys); + if r.is_err() { + eprintln!("{} -> {:?}", msg, &r); + } + + r +} + +//------------------------------------------ + const NODE_HEADER_SIZE: usize = 32; +#[derive(Error, Clone, Debug)] +pub enum BTreeError { + // #[error("io error")] + IoError, // (std::io::Error), // FIXME: we can't clone an io_error + + // #[error("node error: {0}")] + NodeError(String), + + // #[error("value error: {0}")] + ValueError(String), + + // #[error("keys: {0:?}")] + KeyContext(KeyRange, Box), + + // #[error("aggregate: {0:?}")] + Aggregate(Vec), + + // #[error("{0:?}, {1}")] + Path(u64, Box), +} + +fn extract_path(e: &BTreeError) -> (Vec, BTreeError) { + let mut path = Vec::new(); + let mut e = e; + + loop { + match e { + BTreeError::Path(b, next) => { + path.push(*b); + e = next.as_ref(); + } + _ => return (path, e.clone()), + } + } +} + +impl fmt::Display for BTreeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (path, e) = extract_path(self); + match e { + BTreeError::IoError => write!(f, "io error, path{:?}", path), + BTreeError::NodeError(msg) => write!(f, "node error: {}, path{:?}", msg, path), + BTreeError::ValueError(msg) => write!(f, "value error: {}, path{:?}", msg, path), + BTreeError::KeyContext(kr, be) => { + write!(f, "{}, effecting keys {}, path{:?}", be, kr, path) + } + BTreeError::Aggregate(errs) => { + for e in errs { + write!(f, "{}", e)? + } + Ok(()) + } + // Can't happen + BTreeError::Path(_, e) => write!(f, "{}", e), + } + } +} +pub fn node_err(msg: &str) -> BTreeError { + BTreeError::NodeError(msg.to_string()) +} + +fn node_err_s(msg: String) -> BTreeError { + BTreeError::NodeError(msg) +} + +pub fn io_err() -> BTreeError { + BTreeError::IoError +} + +pub fn value_err(msg: String) -> BTreeError { + BTreeError::ValueError(msg) +} + +fn aggregate_error(rs: Vec) -> BTreeError { + BTreeError::Aggregate(rs) +} + +impl BTreeError { + pub fn keys_context(self, keys: &KeyRange) -> BTreeError { + BTreeError::KeyContext(keys.clone(), Box::new(self)) + } +} + +pub type Result = std::result::Result; + +//------------------------------------------ + +#[derive(Debug, Clone)] pub struct NodeHeader { - is_leaf: bool, - nr_entries: u32, - max_entries: u32, - value_size: u32, + pub block: u64, + pub is_leaf: bool, + pub nr_entries: u32, + pub max_entries: u32, + pub value_size: u32, } #[allow(dead_code)] const INTERNAL_NODE: u32 = 1; const LEAF_NODE: u32 = 2; -pub fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { - let (i, _csum) = le_u32(data)?; - let (i, flags) = le_u32(i)?; - let (i, _block) = le_u64(i)?; - let (i, nr_entries) = le_u32(i)?; - let (i, max_entries) = le_u32(i)?; - let (i, value_size) = le_u32(i)?; - let (i, _padding) = le_u32(i)?; +impl Unpack for NodeHeader { + fn disk_size() -> u32 { + 32 + } - Ok(( - i, - NodeHeader { - is_leaf: flags == LEAF_NODE, - nr_entries, - max_entries, - value_size, - }, - )) + fn unpack(data: &[u8]) -> IResult<&[u8], NodeHeader> { + let (i, _csum) = le_u32(data)?; + let (i, flags) = le_u32(i)?; + let (i, block) = le_u64(i)?; + let (i, nr_entries) = le_u32(i)?; + let (i, max_entries) = le_u32(i)?; + let (i, value_size) = le_u32(i)?; + let (i, _padding) = le_u32(i)?; + + Ok(( + i, + NodeHeader { + block, + is_leaf: flags == LEAF_NODE, + nr_entries, + max_entries, + value_size, + }, + )) + } } +#[derive(Clone)] pub enum Node { Internal { header: NodeHeader, @@ -59,17 +367,22 @@ pub enum Node { }, } -pub fn node_err(msg: String) -> Result { - let msg = format!("btree node error: {}", msg); - Err(anyhow!(msg)) +impl Node { + pub fn get_header(&self) -> &NodeHeader { + use Node::*; + match self { + Internal { header, .. } => header, + Leaf { header, .. } => header, + } + } } -pub fn to_any<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { - if let Ok((i, v)) = r { - Ok((i, v)) - } else { - Err(anyhow!("btree node error: parse error")) - } +pub fn convert_result<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { + r.map_err(|_e| node_err("parse error")) +} + +pub fn convert_io_err(r: std::io::Result) -> Result { + r.map_err(|_| io_err()) } pub fn unpack_node( @@ -79,45 +392,52 @@ pub fn unpack_node( ) -> Result> { use nom::multi::count; - let (i, header) = to_any(unpack_node_header(data))?; + let (i, header) = + NodeHeader::unpack(data).map_err(|_e| node_err("couldn't parse node header"))?; if header.is_leaf && header.value_size != V::disk_size() { - return node_err(format!( + return Err(node_err_s(format!( "value_size mismatch: expected {}, was {}", V::disk_size(), header.value_size - )); + ))); } let elt_size = header.value_size + 8; if elt_size as usize * header.max_entries as usize + NODE_HEADER_SIZE > BLOCK_SIZE { - return node_err(format!("max_entries is too large ({})", header.max_entries)); + return Err(node_err_s(format!( + "max_entries is too large ({})", + header.max_entries + ))); } if header.nr_entries > header.max_entries { - return node_err("nr_entries > max_entries".to_string()); + return Err(node_err("nr_entries > max_entries")); } if !ignore_non_fatal { if header.max_entries % 3 != 0 { - return node_err("max_entries is not divisible by 3".to_string()); + return Err(node_err("max_entries is not divisible by 3")); } if !is_root { let min = header.max_entries / 3; if header.nr_entries < min { - return node_err("too few entries".to_string()); + return Err(node_err_s(format!( + "too few entries {}, expected at least {}", + header.nr_entries, min + ))); } } } - let (i, keys) = to_any(count(le_u64, header.nr_entries as usize)(i))?; + let (i, keys) = convert_result(count(le_u64, header.nr_entries as usize)(i))?; let mut last = None; for k in &keys { if let Some(l) = last { if k <= l { - return node_err("keys out of order".to_string()); + return Err(node_err("keys out of order")); } } @@ -125,10 +445,10 @@ pub fn unpack_node( } let nr_free = header.max_entries - header.nr_entries; - let (i, _padding) = to_any(count(le_u64, nr_free as usize)(i))?; + let (i, _padding) = convert_result(count(le_u64, nr_free as usize)(i))?; if header.is_leaf { - let (_i, values) = to_any(count(V::unpack, header.nr_entries as usize)(i))?; + let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?; Ok(Node::Leaf { header, @@ -136,7 +456,7 @@ pub fn unpack_node( values, }) } else { - let (_i, values) = to_any(count(le_u64, header.nr_entries as usize)(i))?; + let (_i, values) = convert_result(count(le_u64, header.nr_entries as usize)(i))?; Ok(Node::Internal { header, keys, @@ -149,13 +469,15 @@ pub fn unpack_node( pub trait NodeVisitor { // &self is deliberately non mut to allow the walker to use multiple threads. - fn visit(&self, header: &NodeHeader, keys: &[u64], values: &[V]) -> Result<()>; + fn visit(&self, keys: &KeyRange, header: &NodeHeader, keys: &[u64], values: &[V]) + -> Result<()>; } #[derive(Clone)] pub struct BTreeWalker { engine: Arc, sm: Arc>, + fails: Arc>>, ignore_non_fatal: bool, } @@ -165,6 +487,7 @@ impl BTreeWalker { let r: BTreeWalker = BTreeWalker { engine, sm: Arc::new(Mutex::new(RestrictedSpaceMap::new(nr_blocks as u64))), + fails: Arc::new(Mutex::new(BTreeMap::new())), ignore_non_fatal, }; r @@ -177,53 +500,116 @@ impl BTreeWalker { ) -> Result { { let sm = sm.lock().unwrap(); - assert_eq!(sm.get_nr_blocks()?, engine.get_nr_blocks()); + assert_eq!(sm.get_nr_blocks().unwrap(), engine.get_nr_blocks()); } Ok(BTreeWalker { engine, sm, + fails: Arc::new(Mutex::new(BTreeMap::new())), ignore_non_fatal, }) } - // Atomically increments the ref count, and returns the _old_ count. - fn sm_inc(&self, b: u64) -> Result { - let mut sm = self.sm.lock().unwrap(); - let count = sm.get(b)?; - sm.inc(b, 1)?; - Ok(count) + fn failed(&self, b: u64) -> Option { + let fails = self.fails.lock().unwrap(); + match fails.get(&b) { + None => None, + Some(e) => Some(e.clone()), + } } - fn walk_nodes(&self, visitor: &NV, bs: &[u64]) -> Result<()> + fn set_fail(&self, b: u64, err: BTreeError) { + // FIXME: should we monitor the size of fails, and abort if too many errors? + let mut fails = self.fails.lock().unwrap(); + fails.insert(b, err); + } + + // Atomically increments the ref count, and returns the _old_ count. + fn sm_inc(&self, b: u64) -> u32 { + let mut sm = self.sm.lock().unwrap(); + let count = sm.get(b).unwrap(); + sm.inc(b, 1).unwrap(); + count + } + + fn build_aggregate(&self, b: u64, errs: Vec) -> Result<()> { + match errs.len() { + 0 => Ok(()), + 1 => { + let e = errs[0].clone(); + self.set_fail(b, e.clone()); + Err(e) + } + _ => { + let e = aggregate_error(errs); + self.set_fail(b, e.clone()); + Err(e) + } + } + } + + fn walk_nodes(&self, visitor: &NV, kr: &[KeyRange], bs: &[u64]) -> Vec where NV: NodeVisitor, V: Unpack, { + let mut errs: Vec = Vec::new(); + let mut blocks = Vec::with_capacity(bs.len()); for b in bs { - if self.sm_inc(*b)? == 0 { + if self.sm_inc(*b) == 0 { + // Node not yet seen blocks.push(*b); + } else { + // This node has already been checked ... + match self.failed(*b) { + None => { + // ... it was clean so we can ignore. + } + Some(e) => { + // ... there was an error + errs.push(e.clone()); + } + } } } - let blocks = self.engine.read_many(&blocks[0..])?; + match self.engine.read_many(&blocks[0..]) { + Err(_) => { + // IO completely failed, error every block + for (i, b) in blocks.iter().enumerate() { + let e = io_err().keys_context(&kr[i]); + errs.push(e.clone()); + self.set_fail(*b, e); + } + } + Ok(rblocks) => { + let mut i = 0; + for rb in rblocks { + match rb { + Err(_) => { + let e = io_err().keys_context(&kr[i]); + errs.push(e.clone()); + self.set_fail(blocks[i], e); + } + Ok(b) => match self.walk_node(visitor, &kr[i], &b, false) { + Err(e) => { + errs.push(e); + } + Ok(()) => {} + }, + } - for b in blocks { - match b { - Err(_e) => { - todo!(); - }, - Ok(b) => { - self.walk_node(visitor, &b, false)?; - }, + i += 1; + } } } - Ok(()) + errs } - fn walk_node(&self, visitor: &NV, b: &Block, is_root: bool) -> Result<()> + fn walk_node_(&self, visitor: &NV, kr: &KeyRange, b: &Block, is_root: bool) -> Result<()> where NV: NodeVisitor, V: Unpack, @@ -232,51 +618,78 @@ impl BTreeWalker { let bt = checksum::metadata_block_type(b.get_data()); if bt != checksum::BT::NODE { - return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); + return Err( + node_err_s(format!("checksum failed for node {}, {:?}", b.loc, bt)) + .keys_context(kr), + ); } let node = unpack_node::(&b.get_data(), self.ignore_non_fatal, is_root)?; match node { - Internal { - header: _h, - keys: _k, - values, - } => { - self.walk_nodes(visitor, &values)?; + Internal { keys, values, .. } => { + let krs = split_key_ranges(&kr, &keys)?; + let errs = self.walk_nodes(visitor, &krs, &values); + return self.build_aggregate(b.loc, errs); } Leaf { header, keys, values, } => { - visitor.visit(&header, &keys, &values)?; + if let Err(e) = visitor.visit(&kr, &header, &keys, &values) { + self.set_fail(b.loc, e.clone()); + return Err(e); + } } } Ok(()) } + fn walk_node(&self, visitor: &NV, kr: &KeyRange, b: &Block, is_root: bool) -> Result<()> + where + NV: NodeVisitor, + V: Unpack, + { + let r = self.walk_node_(visitor, kr, b, is_root); + let r = match r { + Err(e) => Err(BTreeError::Path(b.loc, Box::new(e))), + Ok(v) => Ok(v), + }; + r + } + pub fn walk(&self, visitor: &NV, root: u64) -> Result<()> where NV: NodeVisitor, V: Unpack, { - if self.sm_inc(root)? > 0 { - Ok(()) + if self.sm_inc(root) > 0 { + if let Some(e) = self.failed(root) { + Err(e.clone()) + } else { + Ok(()) + } } else { - let root = self.engine.read(root)?; - self.walk_node(visitor, &root, true) + let root = self.engine.read(root).map_err(|_| io_err())?; + let kr = KeyRange { + start: None, + end: None, + }; + self.walk_node(visitor, &kr, &root, true) } } } //-------------------------------- +/* fn walk_node_threaded( w: Arc, pool: &ThreadPool, visitor: Arc, + kr: &KeyRange, b: &Block, is_root: bool, ) -> Result<()> @@ -288,25 +701,25 @@ where let bt = checksum::metadata_block_type(b.get_data()); if bt != checksum::BT::NODE { - return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); + return Err(node_err_s(format!( + "checksum failed for node {}, {:?}", + b.loc, bt + ))); } let node = unpack_node::(&b.get_data(), w.ignore_non_fatal, is_root)?; match node { - Internal { - header: _h, - keys: _k, - values, - } => { - walk_nodes_threaded(w, pool, visitor, &values)?; + Internal { keys, values, .. } => { + let krs = BTreeWalker::split_key_ranges(&kr, &keys)?; + walk_nodes_threaded(w, pool, visitor, &krs, &values)?; } Leaf { header, keys, values, } => { - visitor.visit(&header, &keys, &values)?; + visitor.visit(kr, &header, &keys, &values)?; } } @@ -317,6 +730,7 @@ fn walk_nodes_threaded( w: Arc, pool: &ThreadPool, visitor: Arc, + krs: &[KeyRange], bs: &[u64], ) -> Result<()> where @@ -330,22 +744,30 @@ where } } - let blocks = w.engine.read_many(&blocks[0..])?; + let rblocks = convert_io_err(w.engine.read_many(&blocks[0..]))?; - for b in blocks { + let mut i = 0; + for b in rblocks { match b { - Err(_e) => { - todo!(); - }, + Err(_) => { + // FIXME: aggregate these + return Err(io_err()); + } Ok(b) => { let w = w.clone(); let visitor = visitor.clone(); + let kr = krs[i].clone(); + pool.execute(move || { - // FIXME: return result - w.walk_node(visitor.as_ref(), &b, false); + let result = w.walk_node(visitor.as_ref(), &kr, &b, false); + if result.is_err() { + todo!(); + } }); } } + + i += 1; } pool.join(); @@ -365,10 +787,28 @@ where if w.sm_inc(root)? > 0 { Ok(()) } else { - let root = w.engine.read(root)?; - walk_node_threaded(w, pool, visitor, &root, true) + let root = convert_io_err(w.engine.read(root))?; + let kr = KeyRange { + start: None, + end: None, + }; + walk_node_threaded(w, pool, visitor, &kr, &root, true) } } +*/ + +pub fn walk_threaded( + w: Arc, + pool: &ThreadPool, + visitor: Arc, + root: u64, +) -> Result<()> +where + NV: NodeVisitor + Send + Sync + 'static, + V: Unpack, +{ + w.walk(visitor.as_ref(), root) +} //------------------------------------------ @@ -385,7 +825,7 @@ impl ValueCollector { } impl NodeVisitor for ValueCollector { - fn visit(&self, _h: &NodeHeader, keys: &[u64], values: &[V]) -> Result<()> { + fn visit(&self, _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], values: &[V]) -> Result<()> { let mut vals = self.values.lock().unwrap(); for n in 0..keys.len() { vals.insert(keys[n], values[n].clone()); @@ -402,7 +842,6 @@ pub fn btree_to_map( ) -> Result> { let walker = BTreeWalker::new(engine, ignore_non_fatal); let visitor = ValueCollector::::new(); - walker.walk(&visitor, root)?; Ok(visitor.values.into_inner().unwrap()) } diff --git a/src/thin/block_time.rs b/src/thin/block_time.rs new file mode 100644 index 0000000..eaf0c96 --- /dev/null +++ b/src/thin/block_time.rs @@ -0,0 +1,40 @@ +use nom::{number::complete::*, IResult}; +use std::fmt; + +use crate::pdata::unpack::*; + +//------------------------------------------ + +#[derive(Clone)] +pub struct BlockTime { + pub block: u64, + pub time: u32, +} + +impl Unpack for BlockTime { + fn disk_size() -> u32 { + 8 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], BlockTime> { + let (i, n) = le_u64(i)?; + let block = n >> 24; + let time = n & ((1 << 24) - 1); + + Ok(( + i, + BlockTime { + block, + time: time as u32, + }, + )) + } +} + +impl fmt::Display for BlockTime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} @ {}", self.block, self.time) + } +} + +//------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index c36c7f8..5548375 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -9,7 +9,7 @@ use threadpool::ThreadPool; use crate::checksum; use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; -use crate::pdata::btree::*; +use crate::pdata::btree::{self, *}; use crate::pdata::space_map::*; use crate::pdata::unpack::*; use crate::report::*; @@ -50,7 +50,7 @@ struct BottomLevelVisitor { //------------------------------------------ impl NodeVisitor for BottomLevelVisitor { - fn visit(&self, _h: &NodeHeader, _k: &[u64], values: &[BlockTime]) -> Result<()> { + fn visit(&self, _kr: &KeyRange, _h: &NodeHeader, _k: &[u64], values: &[BlockTime]) -> btree::Result<()> { // FIXME: do other checks if values.len() == 0 { @@ -67,13 +67,13 @@ impl NodeVisitor for BottomLevelVisitor { if block == start + len { len += 1; } else { - data_sm.inc(start, len)?; + data_sm.inc(start, len).unwrap(); start = block; len = 1; } } - data_sm.inc(start, len)?; + data_sm.inc(start, len).unwrap(); Ok(()) } } @@ -124,14 +124,14 @@ impl<'a> OverflowChecker<'a> { } impl<'a> NodeVisitor for OverflowChecker<'a> { - fn visit(&self, _h: &NodeHeader, keys: &[u64], values: &[u32]) -> Result<()> { + fn visit(&self, _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], values: &[u32]) -> btree::Result<()> { for n in 0..keys.len() { let k = keys[n]; let v = values[n]; - let expected = self.data_sm.get(k)?; + let expected = self.data_sm.get(k).unwrap(); if expected != v { - return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map contains {}.", - k, expected, v)); + return Err(value_err(format!("Bad reference count for data block {}. Expected {}, but space map contains {}.", + k, expected, v))); } } @@ -382,7 +382,7 @@ fn check_mapping_bottom_level( false, )?); - if roots.len() > 64 { + if roots.len() > 64000 { ctx.report.info("spreading load across devices"); for (_thin_id, root) in roots { let data_sm = data_sm.clone(); @@ -390,6 +390,7 @@ fn check_mapping_bottom_level( let v = BottomLevelVisitor { data_sm }; let w = w.clone(); ctx.pool.execute(move || { + // FIXME: propogate errors + share fails. let _r = w.walk(&v, root); }); } @@ -401,6 +402,7 @@ fn check_mapping_bottom_level( let data_sm = data_sm.clone(); let root = *root; let v = Arc::new(BottomLevelVisitor { data_sm }); + // FIXME: propogate errors + share fails. walk_threaded(w, &ctx.pool, v, root)? } } @@ -480,15 +482,16 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { )?; // mapping top level + report.set_sub_title("mapping tree"); let roots = btree_to_map_with_sm::(engine.clone(), metadata_sm.clone(), false, sb.mapping_root)?; // mapping bottom level - report.set_sub_title("mapping tree"); let root = unpack::(&sb.data_sm_root[0..])?; let data_sm = core_sm(root.nr_blocks, nr_devs as u32); check_mapping_bottom_level(&ctx, &metadata_sm, &data_sm, &roots)?; bail_out(&ctx, "mapping tree")?; + eprintln!("checked mapping"); report.set_sub_title("data space map"); let root = unpack::(&sb.data_sm_root[0..])?; @@ -556,7 +559,6 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { let mut stop_progress = stop_progress.lock().unwrap(); *stop_progress = true; } - tid.join(); Ok(()) diff --git a/src/thin/mod.rs b/src/thin/mod.rs index b88bba8..6506a0e 100644 --- a/src/thin/mod.rs +++ b/src/thin/mod.rs @@ -1,3 +1,4 @@ +pub mod block_time; pub mod superblock; pub mod check; pub mod xml; diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index ab9eb3f..73e5b82 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -1,17 +1,27 @@ use crate::io_engine::*; use anyhow::{anyhow, Result}; use nom::{bytes::complete::*, number::complete::*, IResult}; - +use std::fmt; pub const SUPERBLOCK_LOCATION: u64 = 0; //const UUID_SIZE: usize = 16; const SPACE_MAP_ROOT_SIZE: usize = 128; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SuperblockFlags { pub needs_check: bool, } -#[derive(Debug)] +impl fmt::Display for SuperblockFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.needs_check { + write!(f, "NEEDS_CHECK") + } else { + write!(f, "-") + } + } +} + +#[derive(Debug, Clone)] pub struct Superblock { pub flags: SuperblockFlags, pub block: u64, @@ -76,7 +86,9 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { Ok(( i, Superblock { - flags: SuperblockFlags {needs_check: (flags & 0x1) != 0}, + flags: SuperblockFlags { + needs_check: (flags & 0x1) != 0, + }, block, //uuid: uuid[0..UUID_SIZE], version,