From 819fc6d54cc4483231c617435ad548fa81d22635 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 21 Sep 2020 13:53:21 +0100 Subject: [PATCH] [thin_explore] accept a node path on the command line Helpful to examine thin_check failures. --- Cargo.lock | 32 ++++--- Cargo.toml | 1 + src/bin/thin_explore.rs | 188 ++++++++++++++++++++++++++++------------ src/pack/mod.rs | 4 +- src/pack/node_encode.rs | 19 ++-- src/pack/vm.rs | 8 +- src/pdata/btree.rs | 92 ++++++++++++++++++-- src/thin/check.rs | 12 +-- 8 files changed, 259 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 904d37b..4349d0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,9 +79,9 @@ checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" [[package]] name = "cc" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" +checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" [[package]] name = "cfg-if" @@ -136,6 +136,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "data-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d0e2d24e5ee3b23a01de38eefdcd978907890701f08ffffd4cb457ca4ee8d6" + [[package]] name = "duct" version = "0.13.4" @@ -279,9 +285,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", @@ -346,9 +352,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.76" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] name = "log" @@ -367,11 +373,12 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "miniz_oxide" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" dependencies = [ "adler", + "autocfg", ] [[package]] @@ -500,9 +507,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175c513d55719db99da20232b06cda8bab6b83ec2d04e3283edf0213c37c1a29" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" dependencies = [ "unicode-xid", ] @@ -667,9 +674,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" +checksum = "6690e3e9f692504b941dc6c3b188fd28df054f7fb8469ab40680df52fdcc842b" dependencies = [ "proc-macro2", "quote", @@ -740,6 +747,7 @@ dependencies = [ "byteorder", "clap", "crc32c", + "data-encoding", "duct", "fixedbitset", "flate2", diff --git a/Cargo.toml b/Cargo.toml index a86f0d8..74836d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ base64 = "0.12" byteorder = "1.3" clap = "2.33" crc32c = "0.4" +data-encoding = "2.3" duct = "0.13" fixedbitset = "0.3" futures = "0.3" diff --git a/src/bin/thin_explore.rs b/src/bin/thin_explore.rs index 2ff1362..efcb1e6 100644 --- a/src/bin/thin_explore.rs +++ b/src/bin/thin_explore.rs @@ -3,7 +3,7 @@ extern crate clap; use anyhow::{anyhow, Result}; use clap::{App, Arg}; use std::fmt; -use std::io::{self, Read, Write}; +use std::io::{self, Write}; use std::path::Path; use std::sync::mpsc; use std::sync::{ @@ -13,19 +13,17 @@ use std::sync::{ use std::thread; use std::time::Duration; -use termion::clear; -use termion::color; use termion::event::Key; use termion::input::TermRead; use termion::raw::IntoRawMode; use tui::{ - backend::{Backend, TermionBackend}, + backend::{TermionBackend}, buffer::Buffer, layout::{Constraint, Direction, Layout, Rect}, style::{Color, Modifier, Style}, terminal::Frame, - text::{Span, Spans}, + text::{Span}, widgets::{Block, Borders, List, ListItem, ListState, Row, StatefulWidget, Table, Widget}, Terminal, }; @@ -47,7 +45,6 @@ pub struct Events { rx: mpsc::Receiver>, input_handle: thread::JoinHandle<()>, ignore_exit_key: Arc, - tick_handle: thread::JoinHandle<()>, } #[derive(Debug, Clone, Copy)] @@ -92,20 +89,10 @@ impl Events { }) }; - let tick_handle = { - thread::spawn(move || loop { - if tx.send(Event::Tick).is_err() { - break; - } - thread::sleep(config.tick_rate); - }) - }; - Events { rx, ignore_exit_key, input_handle, - tick_handle, } } @@ -138,7 +125,7 @@ fn ls_next(ls: &mut ListState, max: usize) { ls.select(Some(i)); } -fn ls_previous(ls: &mut ListState, max: usize) { +fn ls_previous(ls: &mut ListState) { let i = match ls.selected() { Some(i) => { if i == 0 { @@ -185,7 +172,6 @@ impl Widget for SBWidget { format!("{}k", sb.data_block_size * 2), ]; - let row_style = Style::default().fg(Color::White); let table = Table::new( ["Field", "Value"].iter(), vec![ @@ -226,6 +212,7 @@ struct HeaderWidget<'a> { impl<'a> Widget for HeaderWidget<'a> { fn render(self, area: Rect, buf: &mut Buffer) { let hdr = &self.hdr; + let block = ["block".to_string(), format!("{}", hdr.block)]; let kind = [ "type".to_string(), match hdr.is_leaf { @@ -237,10 +224,10 @@ impl<'a> Widget for HeaderWidget<'a> { let max_entries = ["max_entries".to_string(), format!("{}", hdr.max_entries)]; let value_size = ["value size".to_string(), format!("{}", hdr.value_size)]; - let row_style = Style::default().fg(Color::White); let table = Table::new( ["Field", "Value"].iter(), vec![ + Row::Data(block.iter()), Row::Data(kind.iter()), Row::Data(nr_entries.iter()), Row::Data(max_entries.iter()), @@ -258,10 +245,12 @@ impl<'a> Widget for HeaderWidget<'a> { } } +/* fn read_node_header(engine: &dyn IoEngine, loc: u64) -> Result { let b = engine.read(loc)?; unpack(&b.get_data()).map_err(|_| anyhow!("couldn't unpack btree header")) } +*/ fn read_node(engine: &dyn IoEngine, loc: u64) -> Result> { let b = engine.read(loc)?; @@ -346,19 +335,16 @@ struct NodeWidget<'a, V: Unpack + Adjacent + Clone> { fn mk_item<'a, V: fmt::Display>(k: u64, v: &V, len: usize) -> ListItem<'a> { if len > 1 { - ListItem::new(Span::raw(format!( - "[{}..{}] -> {}", - k, k + len as u64, v - ))) + ListItem::new(Span::raw(format!("{} x {} -> {}", k, len as u64, v))) } else { ListItem::new(Span::raw(format!("{} -> {}", k, v))) } } -fn mk_items<'a, V>(keys: &[u64], values: &[V], selected: usize) -> (Vec>, usize) - where - V: Adjacent + Copy + fmt::Display - { +fn mk_items<'a, V>(keys: &[u64], values: &[V], selected: usize) -> (Vec>, usize) +where + V: Adjacent + Copy + fmt::Display, +{ let mut items = Vec::new(); let bkeys = &keys[0..selected]; let key = keys[selected]; @@ -400,7 +386,7 @@ impl<'a, V: Unpack + fmt::Display + Adjacent + Copy> StatefulWidget for NodeWidg }; hdr.render(chunks[0], buf); - let mut items: Vec; + let items: Vec; let i: usize; let selected = state.selected().unwrap(); let mut state = ListState::default(); @@ -439,11 +425,14 @@ enum Action { PopPanel, } +use Action::*; + type Frame_<'a, 'b> = Frame<'a, TermionBackend>>>; trait Panel { fn render(&mut self, area: Rect, f: &mut Frame_); fn input(&mut self, k: Key) -> Option; + fn path_action(&mut self, child: u64) -> Option; } struct SBPanel { @@ -459,9 +448,17 @@ impl Panel for SBPanel { f.render_widget(w, area); } - fn input(&mut self, k: Key) -> Option { + fn input(&mut self, _k: Key) -> Option { None } + + fn path_action(&mut self, child: u64) -> Option { + if child == self.sb.mapping_root { + Some(PushTopLevel(child)) + } else { + None + } + } } struct TopLevelPanel { @@ -501,23 +498,48 @@ impl Panel for TopLevelPanel { None } Key::Char('k') | Key::Up => { - ls_previous(&mut self.state, self.nr_entries); + ls_previous(&mut self.state); None } Key::Char('l') | Key::Right => match &self.node { btree::Node::Internal { values, .. } => { - Some(Action::PushTopLevel(values[self.state.selected().unwrap()])) + Some(PushTopLevel(values[self.state.selected().unwrap()])) } btree::Node::Leaf { values, keys, .. } => { let index = self.state.selected().unwrap(); - Some(Action::PushBottomLevel(keys[index] as u32, values[index])) + Some(PushBottomLevel(keys[index] as u32, values[index])) } }, - Key::Char('h') | Key::Left => Some(Action::PopPanel), + Key::Char('h') | Key::Left => Some(PopPanel), _ => None, } } + + fn path_action(&mut self, child: u64) -> Option { + match &self.node { + btree::Node::Internal { values, .. } => { + for i in 0..values.len() { + if values[i] == child { + self.state.select(Some(i)); + return Some(PushTopLevel(child)); + } + } + + return None; + } + btree::Node::Leaf { keys, values, .. } => { + for i in 0..values.len() { + if values[i] == child { + self.state.select(Some(i)); + return Some(PushBottomLevel(keys[i] as u32, child)); + } + } + + return None; + } + } + } } struct BottomLevelPanel { @@ -559,43 +581,95 @@ impl Panel for BottomLevelPanel { None } Key::Char('k') | Key::Up => { - ls_previous(&mut self.state, self.nr_entries); + ls_previous(&mut self.state); None } Key::Char('l') | Key::Right => match &self.node { - btree::Node::Internal { values, .. } => Some(Action::PushBottomLevel( + btree::Node::Internal { values, .. } => Some(PushBottomLevel( self.thin_id, values[self.state.selected().unwrap()], )), _ => None, }, - Key::Char('h') | Key::Left => Some(Action::PopPanel), + Key::Char('h') | Key::Left => Some(PopPanel), _ => None, } } + + fn path_action(&mut self, child: u64) -> Option { + match &self.node { + btree::Node::Internal { values, .. } => { + for i in 0..values.len() { + if values[i] == child { + self.state.select(Some(i)); + return Some(PushBottomLevel(self.thin_id, child)); + } + } + + return None; + } + btree::Node::Leaf { .. } => None, + } + } } //------------------------------------ -fn explore(path: &Path) -> Result<()> { +fn perform_action( + panels: &mut Vec>, + engine: &dyn IoEngine, + action: Action, +) -> Result<()> { + match action { + PushTopLevel(b) => { + let node = read_node::(engine, b)?; + panels.push(Box::new(TopLevelPanel::new(node))); + } + PushBottomLevel(thin_id, b) => { + let node = read_node::(engine, b)?; + panels.push(Box::new(BottomLevelPanel::new(thin_id, node))); + } + PopPanel => { + if panels.len() > 2 { + panels.pop(); + } + } + }; + Ok(()) +} + +fn explore(path: &Path, node_path: Option>) -> Result<()> { let stdout = io::stdout(); let mut stdout = stdout.lock().into_raw_mode()?; - write!(stdout, "{}", termion::clear::All); + write!(stdout, "{}", termion::clear::All)?; let backend = TermionBackend::new(stdout); let mut terminal = Terminal::new(backend)?; - let path = std::path::Path::new("bz1763895/meta.bin"); let engine = SyncIoEngine::new(&path, 1, false)?; let mut panels: Vec> = Vec::new(); - let sb = read_superblock(&engine, 0)?; - panels.push(Box::new(SBPanel { sb: sb.clone() })); + if let Some(path) = node_path { + assert_eq!(path[0], 0); + let sb = read_superblock(&engine, path[0])?; + panels.push(Box::new(SBPanel { sb })); + for b in &path[1..] { + let action = panels.last_mut().unwrap().path_action(*b); + if let Some(action) = action { + perform_action(&mut panels, &engine, action)?; + } else { + return Err(anyhow!("bad node path: couldn't find child node {}", b)); + } + } + } else { + let sb = read_superblock(&engine, 0)?; + panels.push(Box::new(SBPanel { sb: sb.clone() })); - let node = read_node::(&engine, sb.mapping_root)?; - panels.push(Box::new(TopLevelPanel::new(node))); + let node = read_node::(&engine, sb.mapping_root)?; + panels.push(Box::new(TopLevelPanel::new(node))); + } let events = Events::new(); @@ -626,18 +700,8 @@ fn explore(path: &Path) -> Result<()> { match key { Key::Char('q') => break 'main, _ => match active_panel.input(key) { - Some(Action::PushTopLevel(b)) => { - let node = read_node::(&engine, b)?; - panels.push(Box::new(TopLevelPanel::new(node))); - } - Some(Action::PushBottomLevel(thin_id, b)) => { - let node = read_node::(&engine, b)?; - panels.push(Box::new(BottomLevelPanel::new(thin_id, node))); - } - Some(Action::PopPanel) => { - if panels.len() > 2 { - panels.pop(); - } + Some(action) => { + perform_action(&mut panels, &engine, action)?; } _ => {} }, @@ -645,6 +709,8 @@ fn explore(path: &Path) -> Result<()> { } } + events.input_handle.join().unwrap(); + Ok(()) } @@ -654,6 +720,13 @@ fn main() -> Result<()> { let parser = App::new("thin_explore") .version(thinp::version::TOOLS_VERSION) .about("A text user interface for examining thin metadata.") + .arg( + Arg::with_name("NODE_PATH") + .help("Pass in a node path as output by thin_check") + .short("p") + .long("node-path") + .value_name("NODE_PATH"), + ) .arg( Arg::with_name("INPUT") .help("Specify the input device to check") @@ -662,9 +735,12 @@ fn main() -> Result<()> { ); let matches = parser.get_matches(); + let node_path = matches + .value_of("NODE_PATH") + .map(|text| btree::decode_node_path(text).unwrap()); let input_file = Path::new(matches.value_of("INPUT").unwrap()); - explore(&input_file) + explore(&input_file, node_path) } //------------------------------------ diff --git a/src/pack/mod.rs b/src/pack/mod.rs index e8ed451..27df5f2 100644 --- a/src/pack/mod.rs +++ b/src/pack/mod.rs @@ -1,5 +1,5 @@ +pub mod node_encode; pub mod toplevel; +pub mod vm; mod delta_list; -mod node_encode; -mod vm; diff --git a/src/pack/node_encode.rs b/src/pack/node_encode.rs index a8a54a6..466fd40 100644 --- a/src/pack/node_encode.rs +++ b/src/pack/node_encode.rs @@ -1,5 +1,5 @@ -use thiserror::Error; use std::{io, io::Write}; +use thiserror::Error; use nom::{bytes::complete::*, number::complete::*, IResult}; @@ -23,7 +23,7 @@ fn nom_to_pr(r: IResult<&[u8], T>) -> PResult<(&[u8], T)> { } fn io_to_pr(r: io::Result) -> PResult { - r.map_err(|source| PackError::WriteError {source}) + r.map_err(|source| PackError::WriteError { source }) } //------------------------------------------- @@ -36,7 +36,7 @@ fn run64(i: &[u8], count: usize) -> IResult<&[u8], Vec> { struct NodeSummary { is_leaf: bool, max_entries: usize, - value_size: usize + value_size: usize, } fn summarise_node(data: &[u8]) -> IResult<&[u8], NodeSummary> { @@ -47,11 +47,14 @@ fn summarise_node(data: &[u8]) -> IResult<&[u8], NodeSummary> { let (i, max_entries) = le_u32(i)?; let (i, value_size) = le_u32(i)?; let (i, _padding) = le_u32(i)?; - Ok((i, NodeSummary { - is_leaf: flags == 2, - max_entries: max_entries as usize, - value_size: value_size as usize, - })) + Ok(( + i, + NodeSummary { + is_leaf: flags == 2, + max_entries: max_entries as usize, + value_size: value_size as usize, + }, + )) } pub fn pack_btree_node(w: &mut W, data: &[u8]) -> PResult<()> { diff --git a/src/pack/vm.rs b/src/pack/vm.rs index 8e3565a..8680a43 100644 --- a/src/pack/vm.rs +++ b/src/pack/vm.rs @@ -206,7 +206,7 @@ fn unpack_with_width(r: &mut R, nibble: u8) -> io::Result { Ok(v) } -fn unpack_u64s(r: &mut R, count: usize) -> io::Result> { +pub fn unpack_u64s(r: &mut R, count: usize) -> io::Result> { let mut v = Vec::with_capacity(count); for _ in 0..count { let n = r.read_u64::()?; @@ -215,13 +215,13 @@ fn unpack_u64s(r: &mut R, count: usize) -> io::Result> { Ok(v) } -struct VM { +pub struct VM { base: u64, bytes_written: usize, } impl VM { - fn new() -> VM { + pub fn new() -> VM { VM { base: 0, bytes_written: 0, @@ -356,7 +356,7 @@ impl VM { } // Runs until at least a number of bytes have been emitted. Returns nr emitted. - fn exec( + pub fn exec( &mut self, r: &mut R, w: &mut W, diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 048f88c..c7d9b78 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -1,14 +1,18 @@ +use anyhow::{anyhow}; +use byteorder::{ReadBytesExt, WriteBytesExt}; use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; use std::fmt; use std::sync::{Arc, Mutex}; use thiserror::Error; use threadpool::ThreadPool; +use data_encoding::BASE64; use crate::checksum; use crate::io_engine::*; use crate::pdata::space_map::*; use crate::pdata::unpack::*; +use crate::pack::vm; // FIXME: check that keys are in ascending order between nodes. @@ -188,7 +192,7 @@ fn split_one(path: &Vec, kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRan } } -fn split_key_ranges_(path: &Vec, kr: &KeyRange, keys: &[u64]) -> Result> { +fn split_key_ranges(path: &Vec, kr: &KeyRange, keys: &[u64]) -> Result> { let mut krs = Vec::with_capacity(keys.len()); if keys.len() == 0 { @@ -212,14 +216,84 @@ fn split_key_ranges_(path: &Vec, kr: &KeyRange, keys: &[u64]) -> Result, kr: &KeyRange, keys: &[u64]) -> Result> { - let msg = format!("split: {:?} at {:?}", &kr, &keys); - let r = split_key_ranges_(path, kr, keys); - if r.is_err() { - eprintln!("{} -> {:?}", msg, &r); - } +//------------------------------------------ - r +pub fn encode_node_path(path: &[u64]) -> String { + let mut buffer: Vec = Vec::with_capacity(128); + let mut cursor = std::io::Cursor::new(&mut buffer); + assert!(path.len() < 256); + + // The first entry is normally the superblock (0), so we + // special case this. + if path[0] == 0 { + let count = ((path.len() as u8) - 1) << 1; + cursor.write_u8(count as u8).unwrap(); + vm::pack_u64s(&mut cursor, &path[1..]).unwrap(); + } else { + let count = ((path.len() as u8) << 1) | 1; + cursor.write_u8(count as u8).unwrap(); + vm::pack_u64s(&mut cursor, path).unwrap(); + } + + BASE64.encode(&buffer) +} + +pub fn decode_node_path(text: &str) -> anyhow::Result> { + let mut buffer = vec![0; 128]; + let bytes = &mut buffer[0..BASE64.decode_len(text.len()).unwrap()]; + let len = BASE64.decode_mut(text.as_bytes(), &mut bytes[0..]).map_err(|_| anyhow!("bad node path. Unable to base64 decode."))?; + + let mut input = std::io::Cursor::new(bytes); + + let mut count = input.read_u8()?; + let mut prepend_zero = false; + if (count & 0x1) == 0 { + // Implicit 0 as first entry + prepend_zero = true; + } + count >>= 1; + + let count = count as usize; + if count == 0 { + return Ok(Vec::new()); + } + + let mut output = Vec::with_capacity(count * 8); + let mut cursor = std::io::Cursor::new(&mut output); + + let mut vm = vm::VM::new(); + let written = vm.exec(&mut input, &mut cursor, count * 8)?; + assert_eq!(written, count * 8); + + let mut cursor = std::io::Cursor::new(&mut output); + let mut path = vm::unpack_u64s(&mut cursor, count)?; + + if prepend_zero { + let mut full_path = vec![0u64]; + full_path.append(&mut path); + Ok(full_path) + } else { + Ok(path) + } +} + +#[test] +fn test_encode_path() { + struct Test(Vec); + + let tests = vec![ + Test(vec![]), + Test(vec![1]), + Test(vec![1, 2]), + Test(vec![1, 2, 3, 4]), + ]; + + for t in tests { + let encoded = encode_node_path(&t.0[0..]); + eprintln!("encoded = '{}'", &encoded); + let decoded = decode_node_path(&encoded).unwrap(); + assert_eq!(decoded, &t.0[0..]); + } } //------------------------------------------ @@ -260,7 +334,7 @@ impl fmt::Display for BTreeError { } Ok(()) } - BTreeError::Path(path, e) => write!(f, "{} @{:?}", e, path), + BTreeError::Path(path, e) => write!(f, "{} {}", e, encode_node_path(path)), } } } diff --git a/src/thin/check.rs b/src/thin/check.rs index e9f64fd..72db4cd 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -56,11 +56,11 @@ impl NodeVisitor for BottomLevelVisitor { //------------------------------------------ #[derive(Clone, Copy)] -struct DeviceDetail { - mapped_blocks: u64, - transaction_id: u64, - creation_time: u32, - snapshotted_time: u32, +pub struct DeviceDetail { + pub mapped_blocks: u64, + pub transaction_id: u64, + pub creation_time: u32, + pub snapshotted_time: u32, } impl Unpack for DeviceDetail { @@ -543,7 +543,7 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { let mut stop_progress = stop_progress.lock().unwrap(); *stop_progress = true; } - tid.join(); + tid.join().unwrap(); Ok(()) }