From e65d2dec6fc91a38b0da2eccb2f0aba7eb0f2e14 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 11 Aug 2020 13:22:14 +0100 Subject: [PATCH] [thin_check (rust)] Add progress bar --- Cargo.lock | 70 ++++++++++++++++++++++++++ Cargo.toml | 1 + src/pdata/space_map.rs | 32 ++++++++++-- src/thin/check.rs | 112 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 201 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdd3687..e6d5c20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,6 +98,23 @@ dependencies = [ "vec_map", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi", + "winapi-util", +] + [[package]] name = "crc32c" version = "0.4.0" @@ -125,6 +142,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "env_logger" version = "0.7.1" @@ -268,6 +291,18 @@ dependencies = [ "libc", ] +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "io-uring" version = "0.3.5" @@ -387,6 +422,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + [[package]] name = "once_cell" version = "1.4.0" @@ -630,6 +671,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -652,6 +712,7 @@ dependencies = [ "fixedbitset", "flate2", "futures", + "indicatif", "io-uring", "json", "libc", @@ -759,6 +820,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 63de8df..86626a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ fixedbitset = "0.3" futures = "0.3" flate2 = "1.0" io-uring = "0.3" +indicatif = "0.15" libc = "0.2.71" nix = "0.17" nom = "5.1" diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index a111c86..1ec9d34 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -156,6 +156,7 @@ impl Unpack for Bitmap { pub trait SpaceMap { fn get_nr_blocks(&self) -> Result; + fn get_nr_allocated(&self) -> Result; fn get(&self, b: u64) -> Result; fn inc(&mut self, begin: u64, len: u64) -> Result<()>; } @@ -163,6 +164,7 @@ pub trait SpaceMap { //------------------------------------------ pub struct CoreSpaceMap { + nr_allocated: u64, counts: Vec, } @@ -172,6 +174,7 @@ where { pub fn new(nr_entries: u64) -> CoreSpaceMap { CoreSpaceMap { + nr_allocated: 0, counts: vec![V::default(); nr_entries as usize], } } @@ -179,19 +182,29 @@ where impl SpaceMap for CoreSpaceMap where - V: Copy + Default + std::ops::AddAssign + From + Into, + V: Copy + Default + Eq + std::ops::AddAssign + From + Into, { fn get_nr_blocks(&self) -> Result { Ok(self.counts.len() as u64) } - + + fn get_nr_allocated(&self) -> Result { + Ok(self.nr_allocated) + } + fn get(&self, b: u64) -> Result { Ok(self.counts[b as usize].into()) } fn inc(&mut self, begin: u64, len: u64) -> Result<()> { for b in begin..(begin + len) { - self.counts[b as usize] += V::from(1u8); + if self.counts[b as usize] == V::from(0u8) { + // FIXME: can we get a ref to save dereferencing counts twice? + self.nr_allocated += 1; + self.counts[b as usize] = V::from(1u8); + } else { + self.counts[b as usize] += V::from(1u8); + } } Ok(()) } @@ -213,12 +226,14 @@ pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc RestrictedSpaceMap { RestrictedSpaceMap { + nr_allocated: 0, counts: FixedBitSet::with_capacity(nr_entries as usize), } } @@ -228,7 +243,11 @@ impl SpaceMap for RestrictedSpaceMap { fn get_nr_blocks(&self) -> Result { Ok(self.counts.len() as u64) } - + + fn get_nr_allocated(&self) -> Result { + Ok(self.nr_allocated) + } + fn get(&self, b: u64) -> Result { if self.counts.contains(b as usize) { Ok(1) @@ -239,7 +258,10 @@ impl SpaceMap for RestrictedSpaceMap { fn inc(&mut self, begin: u64, len: u64) -> Result<()> { for b in begin..(begin + len) { - self.counts.insert(b as usize); + if !self.counts.contains(b as usize) { + self.nr_allocated += 1; + self.counts.insert(b as usize); + } } Ok(()) } diff --git a/src/thin/check.rs b/src/thin/check.rs index 5d94ed9..448dad7 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,8 +1,11 @@ use anyhow::{anyhow, Result}; +use indicatif::{ProgressBar, ProgressStyle}; use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; use std::path::Path; +use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError}; use std::sync::{Arc, Mutex}; +use std::{thread, time}; use threadpool::ThreadPool; use crate::checksum; @@ -176,6 +179,86 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { //------------------------------------------ +enum SpinnerCmd { + Complete, + Abort, + Title(String), +} + +struct Spinner { + tx: Sender, + tid: thread::JoinHandle<()>, +} + +impl Spinner { + fn new(sm: Arc>, total_allocated: u64) -> Result { + let (tx, rx) = channel(); + let tid = thread::spawn(move || spinner_thread(sm, total_allocated, rx)); + Ok(Spinner { tx, tid }) + } + + fn complete(self) -> Result<()> { + self.tx.send(SpinnerCmd::Complete)?; + self.tid.join(); + Ok(()) + } + + fn abort(self) -> Result<()> { + self.tx.send(SpinnerCmd::Abort)?; + self.tid.join(); + Ok(()) + } + + fn set_title(&mut self, txt: &str) -> Result<()> { + self.tx.send(SpinnerCmd::Title(txt.to_string()))?; + Ok(()) + } +} + +fn spinner_thread( + sm: Arc>, + total_allocated: u64, + rx: Receiver, +) { + let interval = time::Duration::from_millis(250); + let bar = ProgressBar::new(total_allocated); + loop { + match rx.try_recv() { + Ok(SpinnerCmd::Complete) => { + bar.finish(); + return; + } + Ok(SpinnerCmd::Abort) => { + return; + } + Ok(SpinnerCmd::Title(txt)) => { + let mut fmt = "Checking thin metadata [{bar:40.cyan/blue}] Remaining {eta}, ".to_string(); + fmt.push_str(&txt); + bar.set_style( + ProgressStyle::default_bar() + .template(&fmt) + .progress_chars("=> "), + ); + } + Err(TryRecvError::Disconnected) => { + return; + } + Err(TryRecvError::Empty) => {} + } + + let sm = sm.lock().unwrap(); + let nr_allocated = sm.get_nr_allocated().unwrap(); + drop(sm); + + bar.set_position(nr_allocated); + bar.tick(); + + thread::sleep(interval); + } +} + +//------------------------------------------ + const MAX_CONCURRENT_IO: u32 = 1024; pub struct ThinCheckOptions<'a> { @@ -191,7 +274,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { nr_threads = std::cmp::min(4, num_cpus::get()); engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO)?); } else { - eprintln!("falling back to synchronous io"); + eprintln!("Using synchronous io"); nr_threads = num_cpus::get() * 2; engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?); } @@ -199,30 +282,39 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // superblock let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; + let nr_allocated_metadata; + { + let root = unpack::(&sb.metadata_sm_root[0..])?; + nr_allocated_metadata = root.nr_allocated; + } + // Device details. We read this once to get the number of thin devices, and hence the // maximum metadata ref count. Then create metadata space map, and reread to increment // the ref counts for that metadata. let devs = btree_to_map::(engine.clone(), false, sb.details_root)?; let nr_devs = devs.len(); let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32); + let mut spinner = Spinner::new(metadata_sm.clone(), nr_allocated_metadata)?; + + spinner.set_title("device details tree")?; let _devs = btree_to_map_with_sm::( engine.clone(), metadata_sm.clone(), false, sb.details_root, )?; - println!("found {} devices", nr_devs); // increment superblock { let mut sm = metadata_sm.lock().unwrap(); sm.inc(SUPERBLOCK_LOCATION, 1)?; } - + // mapping top level let roots = btree_to_map::(engine.clone(), false, sb.mapping_root)?; // Check the mappings filling in the data_sm as we go. + spinner.set_title("mapping tree")?; let data_sm; { // FIXME: with a thread pool we need to return errors another way. @@ -232,7 +324,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let root = unpack::(&sb.data_sm_root[0..])?; data_sm = core_sm(root.nr_blocks, nr_devs as u32); - for (thin_id, root) in roots { + for (_thin_id, root) in roots { let mut w = BTreeWalker::new_with_sm(engine.clone(), metadata_sm.clone(), false)?; let data_sm = data_sm.clone(); pool.execute(move || { @@ -244,8 +336,8 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { eprintln!("walk failed {:?}", e); std::process::abort(); } - Ok(result) => { - eprintln!("checked thin_dev {} -> {:?}", thin_id, result); + Ok(_result) => { + //eprintln!("checked thin_dev {} -> {:?}", thin_id, result); } } }); @@ -256,10 +348,10 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // Check the data space map. { + spinner.set_title("data space map")?; let data_sm = data_sm.lock().unwrap(); let root = unpack::(&sb.data_sm_root[0..])?; let nr_data_blocks = root.nr_blocks; - eprintln!("data root: {:?}", root); // overflow btree { @@ -270,7 +362,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // Bitmaps let entries = btree_to_map::(engine.clone(), false, root.bitmap_root)?; - eprintln!("{} index entries", entries.len()); let mut blocks = Vec::new(); for (_k, i) in &entries { @@ -302,7 +393,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { BitmapEntry::Small(actual) => { let expected = data_sm.get(blocknr)?; if actual == 1 && expected == 0 { - eprintln!("Data block {} leaked.", blocknr); + // eprintln!("Data block {} leaked.", blocknr); leaks += 1; } else if actual != expected as u8 { eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", @@ -331,12 +422,15 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { } if fail { + spinner.abort()?; return Err(anyhow!("Inconsistent data space map")); } } // Check the metadata space map. + spinner.set_title("metadata space map")?; + spinner.complete()?; Ok(()) }