[cache_check (rust)] Add more checks

- Report errors
- Support reading partially broken bitset
  - The output is a bitmap of 2-bit entries, indicating availability of bits

src/bin/cache_check.rs

@@ -1,8 +1,12 @@
 extern crate clap;
 extern crate thinp;
 
+use atty::Stream;
 use clap::{App, Arg};
 use std::path::Path;
+use std::sync::Arc;
+
+use thinp::report::*;
 use thinp::cache::check::{check, CacheCheckOptions};
 
 //------------------------------------------
@@ -39,11 +43,26 @@ fn main() {
                 .help("Don't check the discard bitset")
                 .long("skip-discards")
                 .value_name("SKIP_DISCARDS"),
+        )
+        .arg(
+            Arg::with_name("QUIET")
+                .help("Suppress output messages, return only exit code.")
+                .short("q")
+                .long("quiet"),
         );
 
     let matches = parser.get_matches();
     let input_file = Path::new(matches.value_of("INPUT").unwrap());
 
+    let report;
+    if matches.is_present("QUIET") {
+        report = std::sync::Arc::new(mk_quiet_report());
+    } else if atty::is(Stream::Stdout) {
+        report = std::sync::Arc::new(mk_progress_bar_report());
+    } else {
+        report = Arc::new(mk_simple_report());
+    }
+
     let opts = CacheCheckOptions {
         dev: &input_file,
         async_io: false,
@@ -51,6 +70,7 @@ fn main() {
         skip_mappings: matches.is_present("SKIP_MAPPINGS"),
         skip_hints: matches.is_present("SKIP_HINTS"),
         skip_discards: matches.is_present("SKIP_DISCARDS"),
+        report,
     };
 
     if let Err(reason) = check(opts) {

src/cache/check.rs

@@ -1,5 +1,4 @@
 use anyhow::anyhow;
-use fixedbitset::FixedBitSet;
 use std::collections::BTreeSet;
 use std::path::Path;
 use std::sync::{Arc, Mutex};
@@ -11,6 +10,7 @@ use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
 use crate::pdata::array::{self, ArrayBlock, ArrayError};
 use crate::pdata::array_walker::*;
 use crate::pdata::bitset::*;
+use crate::report::*;
 
 //------------------------------------------
 
@@ -102,11 +102,11 @@ mod format2 {
 
     struct Inner {
         seen_oblocks: BTreeSet<u64>,
-        dirty_bits: FixedBitSet,
+        dirty_bits: CheckedBitSet,
     }
 
     impl MappingChecker {
-        pub fn new(nr_origin_blocks: Option<u64>, dirty_bits: FixedBitSet) -> MappingChecker {
+        pub fn new(nr_origin_blocks: Option<u64>, dirty_bits: CheckedBitSet) -> MappingChecker {
             MappingChecker {
                 nr_origin_blocks: if let Some(n) = nr_origin_blocks {n} else {MAX_ORIGIN_BLOCKS},
                 inner: Mutex::new(Inner {
@@ -116,11 +116,11 @@ mod format2 {
             }
         }
 
-        fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> array::Result<()> {
+        fn check_flags(&self, m: &Mapping, dirty_bit: Option<bool>) -> array::Result<()> {
             if (m.flags & !(MappingFlags::Valid as u32)) != 0 {
                 return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags)));
             }
-            if !m.is_valid() && dirty_bit {
+            if !m.is_valid() && dirty_bit.is_some() && dirty_bit.unwrap() {
                 return Err(array::value_err(format!("dirty bit found on an unmapped block")));
             }
             Ok(())
@@ -203,10 +203,12 @@ pub struct CacheCheckOptions<'a> {
     pub skip_mappings: bool,
     pub skip_hints: bool,
     pub skip_discards: bool,
+    pub report: Arc<Report>,
 }
 
-// TODO: thread pool, report
+// TODO: thread pool
 struct Context {
+    report: Arc<Report>,
     engine: Arc<dyn IoEngine + Send + Sync>,
 }
 
@@ -220,7 +222,10 @@ fn mk_context(opts: &CacheCheckOptions) -> anyhow::Result<Context> {
         engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads, false)?);
     }
 
-    Ok(Context { engine })
+    Ok(Context {
+        report: opts.report.clone(),
+        engine,
+    })
 }
 
 fn check_superblock(sb: &Superblock) -> anyhow::Result<()> {
@@ -256,15 +261,25 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> {
         match sb.version {
             1 => {
                 let mut c = format1::MappingChecker::new(nr_origin_blocks);
-                w.walk(&mut c, sb.mapping_root)?;
+                if let Err(e) = w.walk(&mut c, sb.mapping_root) {
+                    ctx.report.fatal(&format!("{}", e));
+                }
             }
             2 => {
-                // FIXME: possibly size truncation on 32-bit targets
-                let mut dirty_bits = FixedBitSet::with_capacity(sb.cache_blocks as usize);
                 // TODO: allow ignore_none_fatal
-                read_bitset(engine.clone(), sb.dirty_root.unwrap(), false, &mut dirty_bits)?;
+                let (dirty_bits, err) = read_bitset(
+                    engine.clone(),
+                    sb.dirty_root.unwrap(),
+                    sb.cache_blocks as usize,
+                    false,
+                );
+                if err.is_some() {
+                    ctx.report.fatal(&format!("{}", err.unwrap()));
+                }
                 let mut c = format2::MappingChecker::new(nr_origin_blocks, dirty_bits);
-                w.walk(&mut c, sb.mapping_root)?;
+                if let Err(e) = w.walk(&mut c, sb.mapping_root) {
+                    ctx.report.fatal(&format!("{}", e));
+                }
             }
             v => {
                 return Err(anyhow!("unsupported metadata version {}", v));
@@ -278,14 +293,23 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> {
         }
         let w = ArrayWalker::new(engine.clone(), false);
         let mut c = HintChecker::new();
-        w.walk(&mut c, sb.hint_root)?;
+        if let Err(e) = w.walk(&mut c, sb.hint_root) {
+            ctx.report.fatal(&format!("{}", e));
+        }
     }
 
     // The discard bitset might not be available if the cache has never been suspended,
     // e.g., a crash of freshly created cache.
     if !opts.skip_discards && sb.discard_root != 0 {
-        let mut discard_bits = FixedBitSet::with_capacity(sb.discard_nr_blocks as usize);
+        let (discard_bits, err) = read_bitset(
-        read_bitset(engine.clone(), sb.discard_root, false, &mut discard_bits)?;
+            engine.clone(),
+            sb.discard_root,
+            sb.cache_blocks as usize,
+            false,
+        );
+        if err.is_some() {
+            ctx.report.fatal(&format!("{}", err.unwrap()));
+        }
     }
 
     Ok(())

src/pdata/array.rs

@@ -1,4 +1,5 @@
 use nom::{multi::count, number::complete::*, IResult};
+use std::fmt;
 use thiserror::Error;
 
 use crate::checksum;
@@ -52,25 +53,43 @@ pub struct ArrayBlock<V: Unpack> {
 
 #[derive(Error, Clone, Debug)]
 pub enum ArrayError {
-    #[error("io_error")]
+    //#[error("io_error")]
     IoError,
 
-    #[error("block error: {0}")]
+    //#[error("block error: {0}")]
     BlockError(String),
 
-    #[error("value error: {0}")]
+    //#[error("value error: {0}")]
     ValueError(String),
 
-    #[error("aggregate: {0:?}")]
+    //#[error("aggregate: {0:?}")]
     Aggregate(Vec<ArrayError>),
 
-    #[error("{0:?}, {1}")]
+    //#[error("{0:?}, {1}")]
     Path(Vec<u64>, Box<ArrayError>),
 
     #[error(transparent)]
     BTreeError(#[from] btree::BTreeError),
 }
 
+impl fmt::Display for ArrayError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ArrayError::IoError => write!(f, "io error"),
+            ArrayError::BlockError(msg) => write!(f, "block error: {}", msg),
+            ArrayError::ValueError(msg) => write!(f, "value error: {}", msg),
+            ArrayError::Aggregate(errs) => {
+                for e in errs {
+                    write!(f, "{}", e)?
+                }
+                Ok(())
+            }
+            ArrayError::Path(path, e) => write!(f, "{} {}", e, btree::encode_node_path(path)),
+            ArrayError::BTreeError(e) => write!(f, "{}", e),
+        }
+    }
+}
+
 pub fn array_block_err(path: &[u64], msg: &str) -> ArrayError {
     ArrayError::Path(
         path.to_vec(),

src/pdata/bitset.rs

@@ -5,21 +5,49 @@ use crate::io_engine::IoEngine;
 use crate::pdata::array::{self, ArrayBlock};
 use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker};
 
-struct BitsetVisitor<'a> {
+pub struct CheckedBitSet {
-    nr_entries: u64,
+    bits: FixedBitSet,
-    bits: Mutex<&'a mut FixedBitSet>,
 }
 
-impl<'a> BitsetVisitor<'a> {
+impl CheckedBitSet {
-    pub fn new(bitset: &'a mut FixedBitSet) -> Self {
+    pub fn with_capacity(bits: usize) -> CheckedBitSet {
-        BitsetVisitor {
+        CheckedBitSet {
-            nr_entries: bitset.len() as u64,
+            bits: FixedBitSet::with_capacity(bits << 1),
-            bits: Mutex::new(bitset),
         }
     }
+
+    pub fn set(&mut self, bit: usize, enabled: bool) {
+        self.bits.set(bit << 1, true);
+        self.bits.set((bit << 1) + 1, enabled);
+    }
+
+    pub fn contains(&self, bit: usize) -> Option<bool> {
+        if !self.bits.contains(bit << 1) {
+            return None;
+        }
+        Some(self.bits.contains((bit << 1) + 1))
+    }
 }
 
-impl<'a> ArrayVisitor<u64> for BitsetVisitor<'a> {
+struct BitsetVisitor {
+    nr_entries: usize,
+    bits: Mutex<CheckedBitSet>,
+}
+
+impl BitsetVisitor {
+    pub fn new(nr_entries: usize) -> Self {
+        BitsetVisitor {
+            nr_entries,
+            bits: Mutex::new(CheckedBitSet::with_capacity(nr_entries)),
+        }
+    }
+
+    pub fn get_bitset(self) -> CheckedBitSet {
+        self.bits.into_inner().unwrap()
+    }
+}
+
+impl ArrayVisitor<u64> for BitsetVisitor {
     fn visit(&self, index: u64, b: ArrayBlock<u64>) -> array::Result<()> {
         let mut begin = index as usize * (b.header.max_entries as usize) << 6;
 
@@ -46,10 +74,15 @@ impl<'a> ArrayVisitor<u64> for BitsetVisitor<'a> {
 pub fn read_bitset(
     engine: Arc<dyn IoEngine + Send + Sync>,
     root: u64,
+    nr_entries: usize,
     ignore_none_fatal: bool,
-    bitset: &mut FixedBitSet,
+)-> (CheckedBitSet, Option<array::ArrayError>) {
-)-> array::Result<()> {
     let w = ArrayWalker::new(engine.clone(), ignore_none_fatal);
-    let mut v = BitsetVisitor::new(bitset);
+    let mut v = BitsetVisitor::new(nr_entries);
-    w.walk(&mut v, root)
+    let err = w.walk(&mut v, root);
+    let e = match err {
+        Ok(()) => None,
+        Err(e) => Some(e),
+    };
+    return (v.get_bitset(), e);
 }