From 13aeefcdeb2e8c7056f67c49db1ee53dda274c44 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 19 Oct 2021 21:53:55 +0800 Subject: [PATCH 01/25] [thin] Fix typo --- thin-provisioning/device_tree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thin-provisioning/device_tree.h b/thin-provisioning/device_tree.h index ffa98ff..e7553d3 100644 --- a/thin-provisioning/device_tree.h +++ b/thin-provisioning/device_tree.h @@ -26,7 +26,7 @@ namespace thin_provisioning { }; inline bool operator==(device_details const& lhs, device_details const& rhs) { - return false; // device_details are not compariable + return false; // device_details are not comparable } inline bool operator!=(device_details const& lhs, device_details const& rhs) { From c8a1da1df9550bd505bbe615a2be72281c1ff63e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 19 Oct 2021 22:35:06 +0800 Subject: [PATCH 02/25] [all] Apply cargo fmt, and fix clippy warnings --- src/bin/pdata_tools.rs | 2 +- src/bin/thin_explore.rs | 6 +-- src/cache/check.rs | 10 ++--- src/cache/superblock.rs | 2 +- src/commands/cache_check.rs | 4 +- src/commands/cache_repair.rs | 4 +- src/commands/cache_restore.rs | 4 +- src/commands/thin_check.rs | 9 ++-- src/commands/thin_metadata_pack.rs | 2 +- src/commands/thin_metadata_unpack.rs | 4 +- src/commands/thin_repair.rs | 4 +- src/commands/thin_restore.rs | 4 +- src/commands/thin_shrink.rs | 2 +- src/commands/utils.rs | 13 ++---- src/file_utils.rs | 17 ++------ src/io_engine.rs | 4 +- src/pack/toplevel.rs | 4 +- src/pack/vm.rs | 4 +- src/pdata/array_walker.rs | 4 +- src/pdata/btree.rs | 4 +- src/pdata/btree_leaf_walker.rs | 6 +-- src/pdata/btree_walker.rs | 10 ++--- src/pdata/space_map_checker.rs | 2 +- src/report.rs | 2 +- src/shrink/toplevel.rs | 12 +++--- src/thin/dump.rs | 2 +- src/thin/restore.rs | 2 +- src/thin/runs.rs | 2 +- tests/cache_check.rs | 9 ++-- tests/cache_repair.rs | 3 +- tests/cache_restore.rs | 3 +- tests/common/cache_xml_generator.rs | 2 +- tests/common/input_arg.rs | 5 +-- tests/common/process.rs | 4 +- tests/common/thin.rs | 2 +- tests/thin_check.rs | 4 +- tests/thin_dump.rs | 64 +++++++++++----------------- tests/thin_metadata_unpack.rs | 14 +++++- tests/thin_repair.rs | 21 ++++----- 39 files changed, 125 insertions(+), 151 deletions(-) diff --git a/src/bin/pdata_tools.rs b/src/bin/pdata_tools.rs index 0ba337c..8dc8cc7 100644 --- a/src/bin/pdata_tools.rs +++ b/src/bin/pdata_tools.rs @@ -23,7 +23,7 @@ fn main_() -> Result<()> { let mut new_args = vec![OsString::from(&name)]; for a in args.into_iter() { - new_args.push(OsString::from(a)); + new_args.push(a); } if name_eq(name, "cache_check") { diff --git a/src/bin/thin_explore.rs b/src/bin/thin_explore.rs index 26776ab..1ca50c4 100644 --- a/src/bin/thin_explore.rs +++ b/src/bin/thin_explore.rs @@ -266,7 +266,7 @@ impl<'a> Widget for HeaderWidget<'a> { fn read_node(engine: &dyn IoEngine, loc: u64) -> Result> { let b = engine.read(loc)?; let path = Vec::new(); - btree::unpack_node(&path, &b.get_data(), true, false) + btree::unpack_node(&path, b.get_data(), true, false) .map_err(|_| anyhow!("couldn't unpack btree node")) } @@ -765,7 +765,7 @@ fn perform_action( } fn explore(path: &Path, node_path: Option>) -> Result<()> { - let engine = SyncIoEngine::new(&path, 1, false)?; + let engine = SyncIoEngine::new(path, 1, false)?; let mut panels: Vec> = Vec::new(); @@ -861,7 +861,7 @@ fn main() -> Result<()> { .map(|text| btree::decode_node_path(text).unwrap()); let input_file = Path::new(matches.value_of("INPUT").unwrap()); - explore(&input_file, node_path) + explore(input_file, node_path) } //------------------------------------ diff --git a/src/cache/check.rs b/src/cache/check.rs index 17632cc..66e9c22 100644 --- a/src/cache/check.rs +++ b/src/cache/check.rs @@ -92,10 +92,10 @@ mod format1 { let mut errs: Vec = Vec::new(); for m in b.values.iter() { - if let Err(e) = self.check_flags(&m) { + if let Err(e) = self.check_flags(m) { errs.push(e); } - if let Err(e) = self.check_oblock(&m) { + if let Err(e) = self.check_oblock(m) { errs.push(e); } } @@ -182,10 +182,10 @@ mod format2 { let cbegin = index as u32 * b.header.max_entries; let cend = cbegin + b.header.nr_entries; for (m, cblock) in b.values.iter().zip(cbegin..cend) { - if let Err(e) = self.check_flags(&m, inner.dirty_bits.contains(cblock as usize)) { + if let Err(e) = self.check_flags(m, inner.dirty_bits.contains(cblock as usize)) { errs.push(e); } - if let Err(e) = self.check_oblock(&m, &mut inner.seen_oblocks) { + if let Err(e) = self.check_oblock(m, &mut inner.seen_oblocks) { errs.push(e); } } @@ -271,7 +271,7 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { let sb = match read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION) { Ok(sb) => sb, Err(e) => { - check_not_xml(&opts.dev, &opts.report); + check_not_xml(opts.dev, &opts.report); return Err(e); } }; diff --git a/src/cache/superblock.rs b/src/cache/superblock.rs index abe9b5d..2df3679 100644 --- a/src/cache/superblock.rs +++ b/src/cache/superblock.rs @@ -96,7 +96,7 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { if version >= 2 { let (m, root) = le_u64(i)?; dirty_root = Some(root); - i = &m; + i = m; } Ok(( diff --git a/src/commands/cache_check.rs b/src/commands/cache_check.rs index 9c74a34..6435425 100644 --- a/src/commands/cache_check.rs +++ b/src/commands/cache_check.rs @@ -7,8 +7,8 @@ use std::process; use std::sync::Arc; use crate::cache::check::{check, CacheCheckOptions}; -use crate::report::*; use crate::commands::utils::*; +use crate::report::*; //------------------------------------------ @@ -76,7 +76,7 @@ pub fn run(args: &[std::ffi::OsString]) { check_file_not_tiny(input_file, &report); let opts = CacheCheckOptions { - dev: &input_file, + dev: input_file, async_io: matches.is_present("ASYNC_IO"), sb_only: matches.is_present("SB_ONLY"), skip_mappings: matches.is_present("SKIP_MAPPINGS"), diff --git a/src/commands/cache_repair.rs b/src/commands/cache_repair.rs index fd569a8..e9d8a03 100644 --- a/src/commands/cache_repair.rs +++ b/src/commands/cache_repair.rs @@ -60,8 +60,8 @@ pub fn run(args: &[std::ffi::OsString]) { check_input_file(input_file, &report); let opts = CacheRepairOptions { - input: &input_file, - output: &output_file, + input: input_file, + output: output_file, async_io: matches.is_present("ASYNC_IO"), report: report.clone(), }; diff --git a/src/commands/cache_restore.rs b/src/commands/cache_restore.rs index f40c86c..f3b6826 100644 --- a/src/commands/cache_restore.rs +++ b/src/commands/cache_restore.rs @@ -51,8 +51,8 @@ pub fn run(args: &[std::ffi::OsString]) { check_output_file(output_file, &report); let opts = CacheRestoreOptions { - input: &input_file, - output: &output_file, + input: input_file, + output: output_file, async_io: matches.is_present("ASYNC_IO"), report: report.clone(), }; diff --git a/src/commands/thin_check.rs b/src/commands/thin_check.rs index 01135d7..f653649 100644 --- a/src/commands/thin_check.rs +++ b/src/commands/thin_check.rs @@ -5,9 +5,9 @@ use std::path::Path; use std::process; use std::sync::Arc; +use crate::commands::utils::*; use crate::io_engine::*; use crate::thin::check::{check, ThinCheckOptions, MAX_CONCURRENT_IO}; -use crate::commands::utils::*; pub fn run(args: &[std::ffi::OsString]) { let parser = App::new("thin_check") @@ -88,7 +88,7 @@ pub fn run(args: &[std::ffi::OsString]) { .index(1), ); - let matches = parser.get_matches_from(args.into_iter()); + let matches = parser.get_matches_from(args.iter()); let input_file = Path::new(matches.value_of("INPUT").unwrap()); let report = mk_report(matches.is_present("QUIET")); @@ -101,14 +101,13 @@ pub fn run(args: &[std::ffi::OsString]) { if matches.is_present("ASYNC_IO") { engine = Arc::new( - AsyncIoEngine::new(&input_file, MAX_CONCURRENT_IO, writable) + AsyncIoEngine::new(input_file, MAX_CONCURRENT_IO, writable) .expect("unable to open input file"), ); } else { let nr_threads = std::cmp::max(8, num_cpus::get() * 2); engine = Arc::new( - SyncIoEngine::new(&input_file, nr_threads, writable) - .expect("unable to open input file"), + SyncIoEngine::new(input_file, nr_threads, writable).expect("unable to open input file"), ); } diff --git a/src/commands/thin_metadata_pack.rs b/src/commands/thin_metadata_pack.rs index 8b4cfa4..cc40189 100644 --- a/src/commands/thin_metadata_pack.rs +++ b/src/commands/thin_metadata_pack.rs @@ -31,7 +31,7 @@ pub fn run(args: &[std::ffi::OsString]) { let report = std::sync::Arc::new(mk_simple_report()); check_input_file(input_file, &report); - if let Err(reason) = crate::pack::toplevel::pack(&input_file, &output_file) { + if let Err(reason) = crate::pack::toplevel::pack(input_file, output_file) { report.fatal(&format!("Application error: {}\n", reason)); exit(1); } diff --git a/src/commands/thin_metadata_unpack.rs b/src/commands/thin_metadata_unpack.rs index 6a2cf8d..4dbb2b2 100644 --- a/src/commands/thin_metadata_unpack.rs +++ b/src/commands/thin_metadata_unpack.rs @@ -1,9 +1,9 @@ extern crate clap; +use crate::file_utils; use clap::{App, Arg}; use std::path::Path; use std::process; -use crate::file_utils; use std::process::exit; @@ -37,7 +37,7 @@ pub fn run(args: &[std::ffi::OsString]) { exit(1); } - if let Err(reason) = crate::pack::toplevel::unpack(&input_file, &output_file) { + if let Err(reason) = crate::pack::toplevel::unpack(input_file, output_file) { eprintln!("Application error: {}", reason); process::exit(1); } diff --git a/src/commands/thin_repair.rs b/src/commands/thin_repair.rs index 39ce159..ee604e7 100644 --- a/src/commands/thin_repair.rs +++ b/src/commands/thin_repair.rs @@ -91,8 +91,8 @@ pub fn run(args: &[std::ffi::OsString]) { }); let opts = ThinRepairOptions { - input: &input_file, - output: &output_file, + input: input_file, + output: output_file, async_io: matches.is_present("ASYNC_IO"), report: report.clone(), overrides: SuperblockOverrides { diff --git a/src/commands/thin_restore.rs b/src/commands/thin_restore.rs index b694304..9742034 100644 --- a/src/commands/thin_restore.rs +++ b/src/commands/thin_restore.rs @@ -51,8 +51,8 @@ pub fn run(args: &[std::ffi::OsString]) { check_output_file(output_file, &report); let opts = ThinRestoreOptions { - input: &input_file, - output: &output_file, + input: input_file, + output: output_file, async_io: matches.is_present("ASYNC_IO"), report: report.clone(), }; diff --git a/src/commands/thin_shrink.rs b/src/commands/thin_shrink.rs index 5d04f85..a19a76c 100644 --- a/src/commands/thin_shrink.rs +++ b/src/commands/thin_shrink.rs @@ -70,7 +70,7 @@ pub fn run(args: &[std::ffi::OsString]) { check_input_file(input_file, &report); if let Err(reason) = - crate::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy) + crate::shrink::toplevel::shrink(input_file, output_file, data_file, size, do_copy) { eprintln!("Application error: {}\n", reason); exit(1); diff --git a/src/commands/utils.rs b/src/commands/utils.rs index 4e5de06..f7a9a2a 100644 --- a/src/commands/utils.rs +++ b/src/commands/utils.rs @@ -10,10 +10,7 @@ use crate::report::*; pub fn check_input_file(input_file: &Path, report: &Report) { if !file_utils::file_exists(input_file) { - report.fatal(&format!( - "Couldn't find input file '{:?}'.", - &input_file - )); + report.fatal(&format!("Couldn't find input file '{:?}'.", &input_file)); exit(1); } @@ -62,15 +59,11 @@ pub fn mk_report(quiet: bool) -> std::sync::Arc { } fn is_xml(line: &[u8]) -> bool { - line.starts_with(b" Result<()> { - let mut file = OpenOptions::new() - .read(true) - .open(input_file)?; + let mut file = OpenOptions::new().read(true).open(input_file)?; let mut data = vec![0; 16]; file.read_exact(&mut data)?; diff --git a/src/file_utils.rs b/src/file_utils.rs index 6c06819..40d1b30 100644 --- a/src/file_utils.rs +++ b/src/file_utils.rs @@ -18,28 +18,19 @@ pub fn is_file_or_blk_(info: FileStat) -> bool { } pub fn file_exists(path: &Path) -> bool { - match stat::stat(path) { - Ok(_) => true, - _ => false, - } + matches!(stat::stat(path), Ok(_)) } pub fn is_file_or_blk(path: &Path) -> bool { match stat::stat(path) { - Ok(info) =>is_file_or_blk_(info), + Ok(info) => is_file_or_blk_(info), _ => false, } } pub fn is_file(path: &Path) -> bool { match stat::stat(path) { - Ok(info) => { - if test_bit(info.st_mode, SFlag::S_IFREG) { - true - } else { - false - } - } + Ok(info) => test_bit(info.st_mode, SFlag::S_IFREG), _ => false, } } @@ -113,5 +104,3 @@ pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result Deref for FileGuard<'a> { type Target = File; fn deref(&self) -> &File { - &self.file.as_ref().expect("empty file guard") + self.file.as_ref().expect("empty file guard") } } @@ -180,7 +180,7 @@ impl SyncIoEngine { fn write_(output: &mut File, b: &Block) -> Result<()> { output.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; - output.write_all(&b.get_data())?; + output.write_all(b.get_data())?; Ok(()) } } diff --git a/src/pack/toplevel.rs b/src/pack/toplevel.rs index 2298cea..0f78868 100644 --- a/src/pack/toplevel.rs +++ b/src/pack/toplevel.rs @@ -66,7 +66,7 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec> { } pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box> { - let nr_blocks = get_nr_blocks(&input_file)?; + let nr_blocks = get_nr_blocks(input_file)?; let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128)); let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs); @@ -122,7 +122,7 @@ where let kind = metadata_block_type(data); if kind != BT::UNKNOWN { z.write_u64::(b)?; - pack_block(&mut z, kind, &data)?; + pack_block(&mut z, kind, data)?; written += 1; if written == 1024 { diff --git a/src/pack/vm.rs b/src/pack/vm.rs index dc55e0e..d5c1602 100644 --- a/src/pack/vm.rs +++ b/src/pack/vm.rs @@ -442,7 +442,7 @@ mod tests { let mut r = Cursor::new(&mut bs); let unpacked = unpack(&mut r, ns.len() * 8).unwrap(); - check_u64s_match(&ns, &unpacked[0..]) + check_u64s_match(ns, &unpacked[0..]) } #[test] @@ -457,7 +457,7 @@ mod tests { ]; for t in &cases { - assert!(check_pack_u64s(&t)); + assert!(check_pack_u64s(t)); } } diff --git a/src/pdata/array_walker.rs b/src/pdata/array_walker.rs index aa314e9..843ba1c 100644 --- a/src/pdata/array_walker.rs +++ b/src/pdata/array_walker.rs @@ -77,7 +77,7 @@ impl<'a, V: Unpack + Copy> NodeVisitor for BlockValueVisitor<'a, V> { for (i, b) in values.iter().enumerate() { // TODO: report indices of array entries based on the type size let mut array_errs = self.array_errs.lock().unwrap(); - array_errs.push(array::io_err(&path, *b).index_context(keys[i])); + array_errs.push(array::io_err(path, *b).index_context(keys[i])); } } Ok(rblocks) => { @@ -85,7 +85,7 @@ impl<'a, V: Unpack + Copy> NodeVisitor for BlockValueVisitor<'a, V> { match rb { Err(_) => { let mut array_errs = self.array_errs.lock().unwrap(); - array_errs.push(array::io_err(&path, values[i]).index_context(keys[i])); + array_errs.push(array::io_err(path, values[i]).index_context(keys[i])); } Ok(b) => { let mut path = path.to_vec(); diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 14105a9..10883d7 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -561,7 +561,7 @@ pub fn unpack_node( if let Some(l) = last { if k <= l { return Err(node_err( - &path, + path, &format!("keys out of order: {} <= {}", k, l), )); } @@ -582,7 +582,7 @@ pub fn unpack_node( values, }) } else { - let (_i, values) = convert_result(&path, count(le_u64, header.nr_entries as usize)(i))?; + let (_i, values) = convert_result(path, count(le_u64, header.nr_entries as usize)(i))?; Ok(Node::Internal { header, keys, diff --git a/src/pdata/btree_leaf_walker.rs b/src/pdata/btree_leaf_walker.rs index b19da99..3da1de3 100644 --- a/src/pdata/btree_leaf_walker.rs +++ b/src/pdata/btree_leaf_walker.rs @@ -132,10 +132,10 @@ impl<'a> LeafWalker<'a> { .keys_context(kr)); } - let node = unpack_node::(path, &b.get_data(), self.ignore_non_fatal, is_root)?; + let node = unpack_node::(path, b.get_data(), self.ignore_non_fatal, is_root)?; if let Internal { keys, values, .. } = node { - let krs = split_key_ranges(path, &kr, &keys)?; + let krs = split_key_ranges(path, kr, &keys)?; if depth == 0 { // it is the lowest internal for i in 0..krs.len() { @@ -187,7 +187,7 @@ impl<'a> LeafWalker<'a> { )); } - let node = unpack_node::(path, &b.get_data(), self.ignore_non_fatal, is_root)?; + let node = unpack_node::(path, b.get_data(), self.ignore_non_fatal, is_root)?; match node { Internal { values, .. } => { diff --git a/src/pdata/btree_walker.rs b/src/pdata/btree_walker.rs index b6dd6b7..b25f204 100644 --- a/src/pdata/btree_walker.rs +++ b/src/pdata/btree_walker.rs @@ -195,11 +195,11 @@ impl BTreeWalker { .keys_context(kr)); } - let node = unpack_node::(path, &b.get_data(), self.ignore_non_fatal, is_root)?; + let node = unpack_node::(path, b.get_data(), self.ignore_non_fatal, is_root)?; match node { Internal { keys, values, .. } => { - let krs = split_key_ranges(path, &kr, &keys)?; + let krs = split_key_ranges(path, kr, &keys)?; let errs = self.walk_nodes(path, visitor, &krs, &values); return self.build_aggregate(b.loc, errs); } @@ -208,7 +208,7 @@ impl BTreeWalker { keys, values, } => { - if let Err(e) = visitor.visit(path, &kr, &header, &keys, &values) { + if let Err(e) = visitor.visit(path, kr, &header, &keys, &values) { let e = BTreeError::Path(path.clone(), Box::new(e)); self.set_fail(b.loc, e.clone()); return Err(e); @@ -286,11 +286,11 @@ where .keys_context(kr)); } - let node = unpack_node::(path, &b.get_data(), w.ignore_non_fatal, is_root)?; + let node = unpack_node::(path, b.get_data(), w.ignore_non_fatal, is_root)?; match node { Internal { keys, values, .. } => { - let krs = split_key_ranges(path, &kr, &keys)?; + let krs = split_key_ranges(path, kr, &keys)?; let errs = walk_nodes_threaded(w.clone(), path, pool, visitor, &krs, &values); return w.build_aggregate(b.loc, errs); } diff --git a/src/pdata/space_map_checker.rs b/src/pdata/space_map_checker.rs index ad5998b..4f284bf 100644 --- a/src/pdata/space_map_checker.rs +++ b/src/pdata/space_map_checker.rs @@ -109,7 +109,7 @@ fn check_low_ref_counts( return Err(anyhow!("Unable to read bitmap block")); } Ok(b) => { - if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { + if checksum::metadata_block_type(b.get_data()) != checksum::BT::BITMAP { report.fatal(&format!( "Index entry points to block ({}) that isn't a bitmap", b.loc diff --git a/src/report.rs b/src/report.rs index 8b0fb3e..37058c8 100644 --- a/src/report.rs +++ b/src/report.rs @@ -117,7 +117,7 @@ impl ReportInner for PBInner { fn set_sub_title(&mut self, txt: &str) { //let mut fmt = "".to_string(); //Checking thin metadata".to_string(); //self.title.clone(); let mut fmt = "Checking thin metadata [{bar:40}] Remaining {eta}, ".to_string(); - fmt.push_str(&txt); + fmt.push_str(txt); self.bar.set_style( ProgressStyle::default_bar() .template(&fmt) diff --git a/src/shrink/toplevel.rs b/src/shrink/toplevel.rs index fb47773..65ffcc5 100644 --- a/src/shrink/toplevel.rs +++ b/src/shrink/toplevel.rs @@ -343,16 +343,16 @@ fn find_first(r: &BlockRange, remaps: &[(BlockRange, BlockRange)]) -> Option { if n == 0 { let (from, _) = &remaps[n]; - overlaps(&r, &from, n) + overlaps(r, from, n) } else if n == remaps.len() { let (from, _) = &remaps[n - 1]; - overlaps(&r, from, n - 1) + overlaps(r, from, n - 1) } else { // Need to check the previous entry let (from, _) = &remaps[n - 1]; - overlaps(&r, &from, n - 1).or_else(|| { + overlaps(r, from, n - 1).or_else(|| { let (from, _) = &remaps[n]; - overlaps(&r, &from, n) + overlaps(r, from, n) }) } } @@ -368,7 +368,7 @@ fn remap(r: &BlockRange, remaps: &[(BlockRange, BlockRange)]) -> Vec let mut remap = Vec::new(); let mut r = r.start..r.end; - if let Some(index) = find_first(&r, &remaps) { + if let Some(index) = find_first(&r, remaps) { let mut index = index; loop { let (from, to) = &remaps[index]; @@ -487,7 +487,7 @@ fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> V rs.push(Region { src: from.start * block_size, dest: to.start * block_size, - len: range_len(&from) * block_size, + len: range_len(from) * block_size, }); } diff --git a/src/thin/dump.rs b/src/thin/dump.rs index 28e21bf..75003dd 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -188,7 +188,7 @@ fn emit_leaf(v: &mut MappingVisitor, b: &Block) -> Result<()> { ))); } - let node = unpack_node::(&path, &b.get_data(), true, true)?; + let node = unpack_node::(&path, b.get_data(), true, true)?; match node { Internal { .. } => { diff --git a/src/thin/restore.rs b/src/thin/restore.rs index ecaa7a0..9ad7b19 100644 --- a/src/thin/restore.rs +++ b/src/thin/restore.rs @@ -153,7 +153,7 @@ impl<'a> Restorer<'a> { }; for (_, leaves) in self.sub_trees.iter() { - release_leaves(self.w, &leaves, &mut value_rc)?; + release_leaves(self.w, leaves, &mut value_rc)?; } Ok(()) diff --git a/src/thin/runs.rs b/src/thin/runs.rs index be8e287..be929ec 100644 --- a/src/thin/runs.rs +++ b/src/thin/runs.rs @@ -125,7 +125,7 @@ impl Gatherer { // Now we need to mark entries that follow a tail as heads. let mut heads = mem::take(&mut self.heads); for t in &self.tails { - if let Some(e) = self.entries.get(&t) { + if let Some(e) = self.entries.get(t) { for n in &e.neighbours { heads.insert(*n); } diff --git a/tests/cache_check.rs b/tests/cache_check.rs index b3e9c9a..d778cca 100644 --- a/tests/cache_check.rs +++ b/tests/cache_check.rs @@ -13,8 +13,7 @@ use common::test_dir::*; //------------------------------------------ -const USAGE: &str = -"cache_check 0.9.0 +const USAGE: &str = "cache_check 0.9.0 USAGE: cache_check [FLAGS] @@ -105,7 +104,10 @@ fn failing_q() -> Result<()> { let md = mk_zeroed_md(&mut td)?; let output = run_fail_raw(cache_check_cmd(args!["-q", &md]))?; assert_eq!(output.stdout.len(), 0); - eprintln!("stderr = '{}'", std::str::from_utf8(&output.stderr).unwrap()); + eprintln!( + "stderr = '{}'", + std::str::from_utf8(&output.stderr).unwrap() + ); assert_eq!(output.stderr.len(), 0); Ok(()) } @@ -128,7 +130,6 @@ fn valid_metadata_passes() -> Result<()> { Ok(()) } - // FIXME: put back in, I don't want to add the --debug- arg to the // tool again, so we should have a little library function for tweaking // metadata version. diff --git a/tests/cache_repair.rs b/tests/cache_repair.rs index 524b376..8a0bb39 100644 --- a/tests/cache_repair.rs +++ b/tests/cache_repair.rs @@ -12,8 +12,7 @@ use common::test_dir::*; //------------------------------------------ -const USAGE: &str = -"cache_repair 0.9.0 +const USAGE: &str = "cache_repair 0.9.0 Repair binary cache metadata, and write it to a different device or file USAGE: diff --git a/tests/cache_restore.rs b/tests/cache_restore.rs index aebc6d1..c6b80aa 100644 --- a/tests/cache_restore.rs +++ b/tests/cache_restore.rs @@ -14,8 +14,7 @@ use common::test_dir::*; //------------------------------------------ -const USAGE: &str = -"cache_restore 0.9.0 +const USAGE: &str = "cache_restore 0.9.0 Convert XML format metadata to binary. USAGE: diff --git a/tests/common/cache_xml_generator.rs b/tests/common/cache_xml_generator.rs index 27c9c6f..1719192 100644 --- a/tests/common/cache_xml_generator.rs +++ b/tests/common/cache_xml_generator.rs @@ -64,7 +64,7 @@ impl XmlGen for CacheGen { let mut cblocks = (0..self.nr_cache_blocks).collect::>(); cblocks.shuffle(&mut rand::thread_rng()); cblocks.truncate(nr_resident as usize); - cblocks.sort(); + cblocks.sort_unstable(); v.mappings_b()?; { diff --git a/tests/common/input_arg.rs b/tests/common/input_arg.rs index 87b1673..2dfae10 100644 --- a/tests/common/input_arg.rs +++ b/tests/common/input_arg.rs @@ -222,9 +222,8 @@ where let wrapper = build_args_fn(P::arg_type())?; wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| { let stderr = run_fail(P::cmd(args))?; - let msg = format!( - "This looks like XML. This tool only checks the binary metadata format.", - ); + let msg = + "This looks like XML. This tool only checks the binary metadata format.".to_string(); assert!(stderr.contains(&msg)); Ok(()) }) diff --git a/tests/common/process.rs b/tests/common/process.rs index 7f6fed6..790b9ef 100644 --- a/tests/common/process.rs +++ b/tests/common/process.rs @@ -57,10 +57,10 @@ impl fmt::Display for Command { fn log_output(output: &process::Output) { use std::str::from_utf8; - if output.stdout.len() > 0 { + if !output.stdout.is_empty() { eprintln!("stdout: \n{}<>", from_utf8(&output.stdout).unwrap()); } - if output.stderr.len() > 0 { + if !output.stderr.is_empty() { eprintln!("stderr: \n{}<>", from_utf8(&output.stderr).unwrap()); } } diff --git a/tests/common/thin.rs b/tests/common/thin.rs index a66ece7..a86a096 100644 --- a/tests/common/thin.rs +++ b/tests/common/thin.rs @@ -118,7 +118,7 @@ pub fn generate_metadata_leaks( pub fn get_needs_check(md: &PathBuf) -> Result { use thinp::thin::superblock::*; - let engine = SyncIoEngine::new(&md, 1, false)?; + let engine = SyncIoEngine::new(md, 1, false)?; let sb = read_superblock(&engine, SUPERBLOCK_LOCATION)?; Ok(sb.flags.needs_check) } diff --git a/tests/thin_check.rs b/tests/thin_check.rs index e22d2b4..14d1c7a 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -145,11 +145,11 @@ fn accepts_quiet() -> Result<()> { let md = mk_valid_md(&mut td)?; let output = run_ok_raw(thin_check_cmd(args!["--quiet", &md]))?; - if output.stdout.len() > 0 { + if !output.stdout.is_empty() { eprintln!("stdout: {:?}", &std::str::from_utf8(&output.stdout)); } - if output.stderr.len() > 0 { + if !output.stderr.is_empty() { eprintln!("stderr: {:?}", &std::str::from_utf8(&output.stderr)); } assert_eq!(output.stdout.len(), 0); diff --git a/tests/thin_dump.rs b/tests/thin_dump.rs index 254c41f..821b96f 100644 --- a/tests/thin_dump.rs +++ b/tests/thin_dump.rs @@ -15,8 +15,7 @@ use common::thin::*; //------------------------------------------ -const USAGE: &str = -"thin_dump 0.9.0 +const USAGE: &str = "thin_dump 0.9.0 Dump thin-provisioning metadata to stdout in XML format USAGE: @@ -178,15 +177,13 @@ fn repair_superblock() -> Result<()> { let before = run_ok_raw(thin_dump_cmd(args![&md]))?; damage_superblock(&md)?; - let after = run_ok_raw(thin_dump_cmd( - args![ - "--repair", - "--transaction-id=1", - "--data-block-size=128", - "--nr-data-blocks=20480", - &md - ], - ))?; + let after = run_ok_raw(thin_dump_cmd(args![ + "--repair", + "--transaction-id=1", + "--data-block-size=128", + "--nr-data-blocks=20480", + &md + ]))?; if !cfg!(feature = "rust_tests") { assert_eq!(after.stderr.len(), 0); } @@ -204,15 +201,12 @@ fn missing_transaction_id() -> Result<()> { let mut td = TestDir::new()?; let md = mk_valid_md(&mut td)?; damage_superblock(&md)?; - let stderr = run_fail( - thin_dump_cmd( - args![ - "--repair", - "--data-block-size=128", - "--nr-data-blocks=20480", - &md - ], - ))?; + let stderr = run_fail(thin_dump_cmd(args![ + "--repair", + "--data-block-size=128", + "--nr-data-blocks=20480", + &md + ]))?; assert!(stderr.contains("transaction id")); Ok(()) } @@ -222,15 +216,12 @@ fn missing_data_block_size() -> Result<()> { let mut td = TestDir::new()?; let md = mk_valid_md(&mut td)?; damage_superblock(&md)?; - let stderr = run_fail( - thin_dump_cmd( - args![ - "--repair", - "--transaction-id=1", - "--nr-data-blocks=20480", - &md - ], - ))?; + let stderr = run_fail(thin_dump_cmd(args![ + "--repair", + "--transaction-id=1", + "--nr-data-blocks=20480", + &md + ]))?; assert!(stderr.contains("data block size")); Ok(()) } @@ -240,15 +231,12 @@ fn missing_nr_data_blocks() -> Result<()> { let mut td = TestDir::new()?; let md = mk_valid_md(&mut td)?; damage_superblock(&md)?; - let stderr = run_fail( - thin_dump_cmd( - args![ - "--repair", - "--transaction-id=1", - "--data-block-size=128", - &md - ], - ))?; + let stderr = run_fail(thin_dump_cmd(args![ + "--repair", + "--transaction-id=1", + "--data-block-size=128", + &md + ]))?; assert!(stderr.contains("nr data blocks")); Ok(()) } diff --git a/tests/thin_metadata_unpack.rs b/tests/thin_metadata_unpack.rs index 2e5d6c3..89b660a 100644 --- a/tests/thin_metadata_unpack.rs +++ b/tests/thin_metadata_unpack.rs @@ -106,8 +106,18 @@ fn end_to_end() -> Result<()> { let mut td = TestDir::new()?; let md_in = mk_valid_md(&mut td)?; let md_out = mk_zeroed_md(&mut td)?; - run_ok(thin_metadata_pack_cmd(args!["-i", &md_in, "-o", "meta.pack"]))?; - run_ok(thin_metadata_unpack_cmd(args!["-i", "meta.pack", "-o", &md_out]))?; + run_ok(thin_metadata_pack_cmd(args![ + "-i", + &md_in, + "-o", + "meta.pack" + ]))?; + run_ok(thin_metadata_unpack_cmd(args![ + "-i", + "meta.pack", + "-o", + &md_out + ]))?; let dump1 = run_ok(thin_dump_cmd(args![&md_in]))?; let dump2 = run_ok(thin_dump_cmd(args![&md_out]))?; diff --git a/tests/thin_repair.rs b/tests/thin_repair.rs index f8264e2..244e4be 100644 --- a/tests/thin_repair.rs +++ b/tests/thin_repair.rs @@ -161,18 +161,15 @@ fn superblock_succeeds() -> Result<()> { } damage_superblock(&md1)?; let md2 = mk_zeroed_md(&mut td)?; - run_ok( - thin_repair_cmd( - args![ - "--transaction-id=1", - "--data-block-size=128", - "--nr-data-blocks=20480", - "-i", - &md1, - "-o", - &md2 - ], - ))?; + run_ok(thin_repair_cmd(args![ + "--transaction-id=1", + "--data-block-size=128", + "--nr-data-blocks=20480", + "-i", + &md1, + "-o", + &md2 + ]))?; let repaired = run_ok_raw(thin_dump_cmd(args![&md2]))?; if !cfg!(feature = "rust_tests") { assert_eq!(repaired.stderr.len(), 0); From 3d36456a36c2ad689df0e026818fbe33cab72604 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 14 Sep 2021 21:37:56 +0800 Subject: [PATCH 03/25] [era_check (rust)] First code drop --- src/bin/pdata_tools.rs | 2 + src/commands/era_check.rs | 84 +++++++++++++++++++++ src/commands/mod.rs | 1 + src/era/check.rs | 153 ++++++++++++++++++++++++++++++++++++++ src/era/mod.rs | 3 + src/era/superblock.rs | 153 ++++++++++++++++++++++++++++++++++++++ src/era/writeset.rs | 35 +++++++++ src/lib.rs | 1 + 8 files changed, 432 insertions(+) create mode 100644 src/commands/era_check.rs create mode 100644 src/era/check.rs create mode 100644 src/era/mod.rs create mode 100644 src/era/superblock.rs create mode 100644 src/era/writeset.rs diff --git a/src/bin/pdata_tools.rs b/src/bin/pdata_tools.rs index 8dc8cc7..1cf2e0a 100644 --- a/src/bin/pdata_tools.rs +++ b/src/bin/pdata_tools.rs @@ -34,6 +34,8 @@ fn main_() -> Result<()> { cache_repair::run(&new_args); } else if name_eq(name, "cache_restore") { cache_restore::run(&new_args); + } else if name_eq(name, "era_check") { + era_check::run(&new_args); } else if name_eq(name, "thin_check") { thin_check::run(&new_args); } else if name_eq(name, "thin_dump") { diff --git a/src/commands/era_check.rs b/src/commands/era_check.rs new file mode 100644 index 0000000..34c1ce8 --- /dev/null +++ b/src/commands/era_check.rs @@ -0,0 +1,84 @@ +extern crate clap; + +use atty::Stream; +use clap::{App, Arg}; +use std::path::Path; +use std::process; +use std::sync::Arc; + +use crate::commands::utils::*; +use crate::era::check::{check, EraCheckOptions}; +use crate::report::*; + +//------------------------------------------ + +pub fn run(args: &[std::ffi::OsString]) { + let parser = App::new("era_check") + .version(crate::version::tools_version()) + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + .arg( + Arg::with_name("AUTO_REPAIR") + .help("Auto repair trivial issues.") + .long("auto-repair"), + ) + .arg( + Arg::with_name("IGNORE_NON_FATAL") + .help("Only return a non-zero exit code if a fatal error is found.") + .long("ignore-non-fatal-errors"), + ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) + .arg( + Arg::with_name("SB_ONLY") + .help("Only check the superblock.") + .long("super-block-only"), + ) + // arguments + .arg( + Arg::with_name("INPUT") + .help("Specify the input device to check") + .required(true) + .index(1), + ); + + let matches = parser.get_matches_from(args); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + + let report = if matches.is_present("QUIET") { + std::sync::Arc::new(mk_quiet_report()) + } else if atty::is(Stream::Stdout) { + std::sync::Arc::new(mk_progress_bar_report()) + } else { + Arc::new(mk_simple_report()) + }; + + check_input_file(input_file, &report); + check_file_not_tiny(input_file, &report); + check_not_xml(input_file, &report); + + let opts = EraCheckOptions { + dev: input_file, + async_io: matches.is_present("ASYNC_IO"), + sb_only: matches.is_present("SB_ONLY"), + ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"), + auto_repair: matches.is_present("AUTO_REPAIR"), + report: report.clone(), + }; + + if let Err(reason) = check(&opts) { + report.fatal(&format!("{}", reason)); + process::exit(1); + } +} + +//------------------------------------------ diff --git a/src/commands/mod.rs b/src/commands/mod.rs index ffc41bb..9ec58f5 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod cache_check; pub mod cache_dump; pub mod cache_repair; pub mod cache_restore; +pub mod era_check; pub mod thin_check; pub mod thin_dump; pub mod thin_metadata_pack; diff --git a/src/era/check.rs b/src/era/check.rs new file mode 100644 index 0000000..ab23ad0 --- /dev/null +++ b/src/era/check.rs @@ -0,0 +1,153 @@ +use anyhow::{anyhow, Result}; +use std::path::Path; +use std::sync::Arc; + +use crate::era::superblock::*; +use crate::era::writeset::*; +use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; +use crate::pdata::array::{self, ArrayBlock, ArrayError}; +use crate::pdata::array_walker::*; +use crate::pdata::bitset::*; +use crate::pdata::btree_walker::*; +use crate::pdata::space_map::*; +use crate::report::*; + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + +//------------------------------------------ + +fn inc_superblock(sm: &ASpaceMap) -> anyhow::Result<()> { + let mut sm = sm.lock().unwrap(); + sm.inc(SUPERBLOCK_LOCATION, 1)?; + Ok(()) +} + +//------------------------------------------ + +struct EraChecker { + current_era: u32, +} + +impl EraChecker { + pub fn new(current_era: u32) -> EraChecker { + EraChecker { current_era } + } +} + +impl ArrayVisitor for EraChecker { + fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { + let mut errs: Vec = Vec::new(); + + let dbegin = index as u32 * b.header.max_entries; + let dend = dbegin + b.header.max_entries; + for (era, dblock) in b.values.iter().zip(dbegin..dend) { + if era > &self.current_era { + errs.push(array::value_err(format!( + "invalid era value at data block {}: {}", + dblock, era + ))); + } + } + + match errs.len() { + 0 => Ok(()), + 1 => Err(errs[0].clone()), + _ => Err(array::aggregate_error(errs)), + } + } +} + +//------------------------------------------ + +pub struct EraCheckOptions<'a> { + pub dev: &'a Path, + pub async_io: bool, + pub sb_only: bool, + pub ignore_non_fatal: bool, + pub auto_repair: bool, + pub report: Arc, +} + +struct Context { + report: Arc, + engine: Arc, +} + +fn mk_context(opts: &EraCheckOptions) -> anyhow::Result { + let engine: Arc; + + if opts.async_io { + engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO, false)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads, false)?); + } + + Ok(Context { + report: opts.report.clone(), + engine, + }) +} + +fn check_superblock(sb: &Superblock) -> anyhow::Result<()> { + if sb.version > 1 { + return Err(anyhow!("unknown superblock version")); + } + Ok(()) +} + +pub fn check(opts: &EraCheckOptions) -> Result<()> { + let ctx = mk_context(opts)?; + let engine = &ctx.engine; + let report = &ctx.report; + let mut fatal = false; + + report.set_title("Checking era metadata"); + + let metadata_sm = core_sm(engine.get_nr_blocks(), u8::MAX as u32); + inc_superblock(&metadata_sm)?; + + let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; + check_superblock(&sb)?; + + if opts.sb_only { + return Ok(()); + } + + let mut path = vec![0]; + let writesets = btree_to_map::( + &mut path, + engine.clone(), + opts.ignore_non_fatal, + sb.writeset_tree_root, + )?; + + for ws in writesets.values() { + let (_bs, err) = read_bitset_with_sm( + engine.clone(), + ws.root, + ws.nr_bits as usize, + metadata_sm.clone(), + opts.ignore_non_fatal, + )?; + if err.is_some() { + ctx.report.fatal(&format!("{}", err.unwrap())); + fatal = true; + } + } + + let w = ArrayWalker::new_with_sm(engine.clone(), metadata_sm.clone(), opts.ignore_non_fatal)?; + let mut c = EraChecker::new(sb.current_era); + if let Err(e) = w.walk(&mut c, sb.era_array_root) { + ctx.report.fatal(&format!("{}", e)); + fatal = true; + } + + if fatal { + Err(anyhow!("fatal errors in metadata")) + } else { + Ok(()) + } +} diff --git a/src/era/mod.rs b/src/era/mod.rs new file mode 100644 index 0000000..59c68bc --- /dev/null +++ b/src/era/mod.rs @@ -0,0 +1,3 @@ +pub mod check; +pub mod superblock; +pub mod writeset; diff --git a/src/era/superblock.rs b/src/era/superblock.rs new file mode 100644 index 0000000..794a4cb --- /dev/null +++ b/src/era/superblock.rs @@ -0,0 +1,153 @@ +use anyhow::{anyhow, Result}; +use byteorder::{LittleEndian, WriteBytesExt}; +use nom::{bytes::complete::*, number::complete::*, IResult}; +use std::io::Cursor; + +use crate::checksum::*; +use crate::era::writeset::Writeset; +use crate::io_engine::*; + +//------------------------------------------ + +pub const SPACE_MAP_ROOT_SIZE: usize = 128; +pub const SUPERBLOCK_LOCATION: u64 = 0; + +const MAGIC: u64 = 0o17660203573; // 0x7EC1077B in hex +const UUID_SIZE: usize = 16; + +//------------------------------------------ + +#[derive(Debug, Clone)] +pub struct SuperblockFlags { + pub clean_shutdown: bool, +} + +#[derive(Debug, Clone)] +pub struct Superblock { + pub flags: SuperblockFlags, + pub block: u64, + pub version: u32, + + pub metadata_sm_root: Vec, + + pub data_block_size: u32, + pub nr_blocks: u32, + + pub current_era: u32, + pub current_writeset: Writeset, + + pub writeset_tree_root: u64, + pub era_array_root: u64, + + pub metadata_snap: u64, +} + +fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { + let (i, _csum) = le_u32(data)?; + let (i, flags) = le_u32(i)?; + let (i, block) = le_u64(i)?; + let (i, _uuid) = take(16usize)(i)?; + let (i, _magic) = le_u64(i)?; + let (i, version) = le_u32(i)?; + + let (i, metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, data_block_size) = le_u32(i)?; + let (i, _metadata_block_size) = le_u32(i)?; + let (i, nr_blocks) = le_u32(i)?; + + let (i, current_era) = le_u32(i)?; + let (i, nr_bits) = le_u32(i)?; + let (i, root) = le_u64(i)?; + + let (i, writeset_tree_root) = le_u64(i)?; + let (i, era_array_root) = le_u64(i)?; + let (i, metadata_snap) = le_u64(i)?; + + Ok(( + i, + Superblock { + flags: SuperblockFlags { + clean_shutdown: (flags & 0x1) != 0, + }, + block, + version, + metadata_sm_root: metadata_sm_root.to_vec(), + data_block_size, + nr_blocks, + current_era, + current_writeset: Writeset { nr_bits, root }, + writeset_tree_root, + era_array_root, + metadata_snap, + }, + )) +} + +pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result { + let b = engine.read(loc)?; + + if metadata_block_type(b.get_data()) != BT::ERA_SUPERBLOCK { + return Err(anyhow!("bad checksum in superblock")); + } + + if let Ok((_, sb)) = unpack(b.get_data()) { + Ok(sb) + } else { + Err(anyhow!("couldn't unpack superblock")) + } +} + +//------------------------------------------ + +fn pack_superblock(sb: &Superblock, w: &mut W) -> Result<()> { + // checksum, which we don't know yet + w.write_u32::(0)?; + + // flags + let mut flags: u32 = 0; + if sb.flags.clean_shutdown { + flags |= 0x1; + } + w.write_u32::(flags)?; + w.write_u64::(sb.block)?; + + w.write_all(&[0; UUID_SIZE])?; + w.write_u64::(MAGIC)?; + w.write_u32::(sb.version)?; + + w.write_all(&sb.metadata_sm_root)?; + w.write_u32::(sb.data_block_size)?; + // metadata block size + w.write_u32::((BLOCK_SIZE >> SECTOR_SHIFT) as u32)?; + w.write_u32::(sb.nr_blocks)?; + + w.write_u32::(sb.current_era)?; + w.write_u32::(sb.current_writeset.nr_bits)?; + w.write_u64::(sb.current_writeset.root)?; + + w.write_u64::(sb.writeset_tree_root)?; + w.write_u64::(sb.era_array_root)?; + + w.write_u64::(sb.metadata_snap)?; + + Ok(()) +} + +pub fn write_superblock(engine: &dyn IoEngine, _loc: u64, sb: &Superblock) -> Result<()> { + let b = Block::zeroed(SUPERBLOCK_LOCATION); + + // pack the superblock + { + let mut cursor = Cursor::new(b.get_data()); + pack_superblock(sb, &mut cursor)?; + } + + // calculate the checksum + write_checksum(b.get_data(), BT::ERA_SUPERBLOCK)?; + + // write + engine.write(&b)?; + Ok(()) +} + +//------------------------------------------ diff --git a/src/era/writeset.rs b/src/era/writeset.rs new file mode 100644 index 0000000..62ab8fd --- /dev/null +++ b/src/era/writeset.rs @@ -0,0 +1,35 @@ +use anyhow::Result; +use byteorder::{LittleEndian, WriteBytesExt}; +use nom::{number::complete::*, IResult}; + +use crate::pdata::unpack::*; + +//------------------------------------------ + +#[derive(Clone, Copy, Debug)] +pub struct Writeset { + pub nr_bits: u32, + pub root: u64, +} + +impl Unpack for Writeset { + fn disk_size() -> u32 { + 12 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], Writeset> { + let (i, nr_bits) = le_u32(i)?; + let (i, root) = le_u64(i)?; + Ok((i, Writeset { nr_bits, root })) + } +} + +impl Pack for Writeset { + fn pack(&self, w: &mut W) -> Result<()> { + w.write_u32::(self.nr_bits)?; + w.write_u64::(self.root)?; + Ok(()) + } +} + +//------------------------------------------ diff --git a/src/lib.rs b/src/lib.rs index 0125836..23fc9c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ extern crate quickcheck_macros; pub mod cache; pub mod checksum; pub mod commands; +pub mod era; pub mod file_utils; pub mod io_engine; pub mod math; From 8a1399e3bb6b60214416e9216ce4f205c15aba8d Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 15 Sep 2021 21:26:18 +0800 Subject: [PATCH 04/25] [cache_dump (rust)] Make use of the --repair option Ignore non-fatal errors, and regenerate unavailable dirty bits if --repair was applied. --- src/cache/dump.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/cache/dump.rs b/src/cache/dump.rs index a5b9309..86579c4 100644 --- a/src/cache/dump.rs +++ b/src/cache/dump.rs @@ -222,7 +222,7 @@ pub fn dump_metadata( engine: Arc, out: &mut dyn MetadataVisitor, sb: &Superblock, - _repair: bool, + repair: bool, ) -> anyhow::Result<()> { let xml_sb = ir::Superblock { uuid: "".to_string(), @@ -236,7 +236,7 @@ pub fn dump_metadata( out.mappings_b()?; let valid_mappings = match sb.version { 1 => { - let w = ArrayWalker::new(engine.clone(), false); + let w = ArrayWalker::new(engine.clone(), repair); let mut emitter = format1::MappingEmitter::new(sb.cache_blocks as usize, out); w.walk(&mut emitter, sb.mapping_root)?; emitter.get_valid() @@ -246,9 +246,8 @@ pub fn dump_metadata( let dirty_bits; if let Some(root) = sb.dirty_root { let (bits, errs) = - read_bitset(engine.clone(), root, sb.cache_blocks as usize, false); - // TODO: allow errors in repair mode - if errs.is_some() { + read_bitset(engine.clone(), root, sb.cache_blocks as usize, repair); + if errs.is_some() && !repair { return Err(anyhow!("errors in bitset {}", errs.unwrap())); } dirty_bits = bits; @@ -258,7 +257,7 @@ pub fn dump_metadata( return Err(anyhow!("format 2 selected, but no dirty bitset present")); } - let w = ArrayWalker::new(engine.clone(), false); + let w = ArrayWalker::new(engine.clone(), repair); let mut emitter = format2::MappingEmitter::new(sb.cache_blocks as usize, dirty_bits, out); w.walk(&mut emitter, sb.mapping_root)?; @@ -272,7 +271,7 @@ pub fn dump_metadata( out.hints_b()?; { - let w = ArrayWalker::new(engine.clone(), false); + let w = ArrayWalker::new(engine.clone(), repair); let mut emitter = HintEmitter::new(out, valid_mappings); w.walk(&mut emitter, sb.hint_root)?; } @@ -296,7 +295,7 @@ pub fn dump(opts: CacheDumpOptions) -> anyhow::Result<()> { } let mut out = xml::XmlWriter::new(writer); - dump_metadata(ctx.engine.clone(), &mut out, &sb, opts.repair) + dump_metadata(ctx.engine, &mut out, &sb, opts.repair) } //------------------------------------------ From 55a81c0b9fe136b4f126b6a2ea49dd6ca073ee2c Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 16 Sep 2021 17:09:47 +0800 Subject: [PATCH 05/25] [era_dump (rust)] First code drop --- src/bin/pdata_tools.rs | 2 + src/commands/era_dump.rs | 79 ++++++++++++++++++ src/commands/mod.rs | 1 + src/era/dump.rs | 176 +++++++++++++++++++++++++++++++++++++++ src/era/ir.rs | 54 ++++++++++++ src/era/mod.rs | 3 + src/era/xml.rs | 93 +++++++++++++++++++++ 7 files changed, 408 insertions(+) create mode 100644 src/commands/era_dump.rs create mode 100644 src/era/dump.rs create mode 100644 src/era/ir.rs create mode 100644 src/era/xml.rs diff --git a/src/bin/pdata_tools.rs b/src/bin/pdata_tools.rs index 1cf2e0a..5081ac0 100644 --- a/src/bin/pdata_tools.rs +++ b/src/bin/pdata_tools.rs @@ -36,6 +36,8 @@ fn main_() -> Result<()> { cache_restore::run(&new_args); } else if name_eq(name, "era_check") { era_check::run(&new_args); + } else if name_eq(name, "era_dump") { + era_dump::run(&new_args); } else if name_eq(name, "thin_check") { thin_check::run(&new_args); } else if name_eq(name, "thin_dump") { diff --git a/src/commands/era_dump.rs b/src/commands/era_dump.rs new file mode 100644 index 0000000..87ec1cf --- /dev/null +++ b/src/commands/era_dump.rs @@ -0,0 +1,79 @@ +extern crate clap; + +use clap::{App, Arg}; +use std::path::Path; +use std::process; + +use crate::commands::utils::*; +use crate::era::dump::{dump, EraDumpOptions}; + +//------------------------------------------ + +pub fn run(args: &[std::ffi::OsString]) { + let parser = App::new("era_dump") + .version(crate::version::tools_version()) + .about("Dump the era metadata to stdout in XML format") + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + .arg( + Arg::with_name("LOGICAL") + .help("Fold any unprocessed write sets into the final era array") + .long("logical"), + ) + .arg( + Arg::with_name("REPAIR") + .help("Repair the metadata whilst dumping it") + .short("r") + .long("repair"), + ) + // options + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output file rather than stdout") + .short("o") + .long("output") + .value_name("FILE"), + ) + // arguments + .arg( + Arg::with_name("INPUT") + .help("Specify the input device to dump") + .required(true) + .index(1), + ); + + let matches = parser.get_matches_from(args); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = if matches.is_present("OUTPUT") { + Some(Path::new(matches.value_of("OUTPUT").unwrap())) + } else { + None + }; + + // Create a temporary report just in case these checks + // need to report anything. + let report = std::sync::Arc::new(crate::report::mk_simple_report()); + check_input_file(input_file, &report); + check_file_not_tiny(input_file, &report); + drop(report); + + let opts = EraDumpOptions { + input: input_file, + output: output_file, + async_io: matches.is_present("ASYNC_IO"), + logical: matches.is_present("LOGICAL"), + repair: matches.is_present("REPAIR"), + }; + + if let Err(reason) = dump(opts) { + eprintln!("{}", reason); + process::exit(1); + } +} + +//------------------------------------------ diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 9ec58f5..c070565 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -3,6 +3,7 @@ pub mod cache_dump; pub mod cache_repair; pub mod cache_restore; pub mod era_check; +pub mod era_dump; pub mod thin_check; pub mod thin_dump; pub mod thin_metadata_pack; diff --git a/src/era/dump.rs b/src/era/dump.rs new file mode 100644 index 0000000..a72bd51 --- /dev/null +++ b/src/era/dump.rs @@ -0,0 +1,176 @@ +use anyhow::anyhow; +use std::fs::File; +use std::io::BufWriter; +use std::io::Write; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use crate::era::ir::{self, MetadataVisitor}; +use crate::era::superblock::*; +use crate::era::writeset::Writeset; +use crate::era::xml; +use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; +use crate::pdata::array::{self, ArrayBlock}; +use crate::pdata::array_walker::*; +use crate::pdata::bitset::read_bitset; +use crate::pdata::btree_walker::btree_to_map; + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + +//----------------------------------------- + +struct EraEmitter<'a> { + emitter: Mutex<&'a mut dyn MetadataVisitor>, +} + +impl<'a> EraEmitter<'a> { + pub fn new(emitter: &'a mut dyn MetadataVisitor) -> EraEmitter { + EraEmitter { + emitter: Mutex::new(emitter), + } + } +} + +impl<'a> ArrayVisitor for EraEmitter<'a> { + fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { + let begin = index as u32 * b.header.max_entries; + let end = begin + b.header.nr_entries; + for (v, block) in b.values.iter().zip(begin..end) { + let era = ir::Era { block, era: *v }; + + self.emitter + .lock() + .unwrap() + .era(&era) + .map_err(|e| array::value_err(format!("{}", e)))?; + } + + Ok(()) + } +} + +//------------------------------------------ + +pub struct EraDumpOptions<'a> { + pub input: &'a Path, + pub output: Option<&'a Path>, + pub async_io: bool, + pub logical: bool, + pub repair: bool, +} + +struct Context { + engine: Arc, +} + +fn mk_context(opts: &EraDumpOptions) -> anyhow::Result { + let engine: Arc; + + if opts.async_io { + engine = Arc::new(AsyncIoEngine::new(opts.input, MAX_CONCURRENT_IO, false)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine = Arc::new(SyncIoEngine::new(opts.input, nr_threads, false)?); + } + + Ok(Context { engine }) +} + +fn dump_writeset( + engine: Arc, + out: &mut dyn MetadataVisitor, + era: u32, + ws: &Writeset, + repair: bool, +) -> anyhow::Result<()> { + let (bits, errs) = read_bitset(engine.clone(), ws.root, ws.nr_bits as usize, repair); + // TODO: deal with broken writeset + if errs.is_some() { + return Err(anyhow!( + "errors in writeset of era {}: {}", + era, + errs.unwrap() + )); + } + + out.writeset_b(&ir::Writeset { + era, + nr_bits: ws.nr_bits, + })?; + for b in 0..ws.nr_bits { + let wbit = ir::WritesetBit { + block: b, + value: bits.contains(b as usize).unwrap_or(false), + }; + out.writeset_bit(&wbit)?; + } + out.writeset_e()?; + + Ok(()) +} + +pub fn dump_metadata( + engine: Arc, + out: &mut dyn MetadataVisitor, + sb: &Superblock, + repair: bool, +) -> anyhow::Result<()> { + let xml_sb = ir::Superblock { + uuid: "".to_string(), + block_size: sb.data_block_size, + nr_blocks: sb.nr_blocks, + current_era: sb.current_era, + }; + out.superblock_b(&xml_sb)?; + + let mut path = vec![0]; + let writesets = + btree_to_map::(&mut path, engine.clone(), repair, sb.writeset_tree_root)?; + for (era, ws) in writesets.iter() { + dump_writeset(engine.clone(), out, *era as u32, ws, repair)?; + } + + out.era_b()?; + let w = ArrayWalker::new(engine.clone(), repair); + let mut emitter = EraEmitter::new(out); + w.walk(&mut emitter, sb.era_array_root)?; + out.era_e()?; + + out.superblock_e()?; + out.eof()?; + + Ok(()) +} + +pub fn dump_metadata_logical( + _engine: Arc, + _out: &mut dyn MetadataVisitor, + _sb: &Superblock, + _repair: bool, +) -> anyhow::Result<()> { + // TODO + Ok(()) +} + +pub fn dump(opts: EraDumpOptions) -> anyhow::Result<()> { + let ctx = mk_context(&opts)?; + let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; + + let writer: Box; + if opts.output.is_some() { + writer = Box::new(BufWriter::new(File::create(opts.output.unwrap())?)); + } else { + writer = Box::new(BufWriter::new(std::io::stdout())); + } + let mut out = xml::XmlWriter::new(writer); + + if opts.logical { + dump_metadata_logical(ctx.engine, &mut out, &sb, opts.repair) + } else { + dump_metadata(ctx.engine, &mut out, &sb, opts.repair) + } +} + +//------------------------------------------ diff --git a/src/era/ir.rs b/src/era/ir.rs new file mode 100644 index 0000000..0dc1342 --- /dev/null +++ b/src/era/ir.rs @@ -0,0 +1,54 @@ +use anyhow::Result; + +//------------------------------------------ + +#[derive(Clone)] +pub struct Superblock { + pub uuid: String, + pub block_size: u32, + pub nr_blocks: u32, + pub current_era: u32, +} + +#[derive(Clone)] +pub struct Writeset { + pub era: u32, + pub nr_bits: u32, +} + +#[derive(Clone)] +pub struct WritesetBit { + pub block: u32, + pub value: bool, +} + +#[derive(Clone)] +pub struct Era { + pub block: u32, + pub era: u32, +} + +//------------------------------------------ + +#[derive(Clone)] +pub enum Visit { + Continue, + Stop, +} + +pub trait MetadataVisitor { + fn superblock_b(&mut self, sb: &Superblock) -> Result; + fn superblock_e(&mut self) -> Result; + + fn writeset_b(&mut self, ws: &Writeset) -> Result; + fn writeset_e(&mut self) -> Result; + fn writeset_bit(&mut self, wbit: &WritesetBit) -> Result; + + fn era_b(&mut self) -> Result; + fn era_e(&mut self) -> Result; + fn era(&mut self, era: &Era) -> Result; + + fn eof(&mut self) -> Result; +} + +//------------------------------------------ diff --git a/src/era/mod.rs b/src/era/mod.rs index 59c68bc..bb9a39f 100644 --- a/src/era/mod.rs +++ b/src/era/mod.rs @@ -1,3 +1,6 @@ pub mod check; +pub mod dump; +pub mod ir; pub mod superblock; pub mod writeset; +pub mod xml; diff --git a/src/era/xml.rs b/src/era/xml.rs new file mode 100644 index 0000000..0b2dee7 --- /dev/null +++ b/src/era/xml.rs @@ -0,0 +1,93 @@ +use anyhow::Result; +use quick_xml::events::{BytesEnd, BytesStart, Event}; +use quick_xml::Writer; +use std::io::Write; + +use crate::era::ir::*; +use crate::xml::*; + +//--------------------------------------- + +pub struct XmlWriter { + w: Writer, +} + +impl XmlWriter { + pub fn new(w: W) -> XmlWriter { + XmlWriter { + w: Writer::new_with_indent(w, 0x20, 2), + } + } +} + +impl MetadataVisitor for XmlWriter { + fn superblock_b(&mut self, sb: &Superblock) -> Result { + let tag = b"superblock"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"uuid", sb.uuid.clone())); + elem.push_attribute(mk_attr(b"block_size", sb.block_size)); + elem.push_attribute(mk_attr(b"nr_blocks", sb.nr_blocks)); + elem.push_attribute(mk_attr(b"current_era", sb.current_era)); + + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn superblock_e(&mut self) -> Result { + self.w + .write_event(Event::End(BytesEnd::borrowed(b"superblock")))?; + Ok(Visit::Continue) + } + + fn writeset_b(&mut self, ws: &Writeset) -> Result { + let tag = b"writeset"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"era", ws.era)); + elem.push_attribute(mk_attr(b"nr_bits", ws.nr_bits)); + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn writeset_e(&mut self) -> Result { + self.w + .write_event(Event::End(BytesEnd::borrowed(b"writeset")))?; + Ok(Visit::Continue) + } + + fn writeset_bit(&mut self, wbit: &WritesetBit) -> Result { + let tag = b"bit"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"block", wbit.block)); + elem.push_attribute(mk_attr(b"value", wbit.value)); + self.w.write_event(Event::Empty(elem))?; + Ok(Visit::Continue) + } + + fn era_b(&mut self) -> Result { + let tag = b"era_array"; + let elem = BytesStart::owned(tag.to_vec(), tag.len()); + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn era_e(&mut self) -> Result { + self.w + .write_event(Event::End(BytesEnd::borrowed(b"era_array")))?; + Ok(Visit::Continue) + } + + fn era(&mut self, era: &Era) -> Result { + let tag = b"era"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"block", era.block)); + elem.push_attribute(mk_attr(b"era", era.era)); + self.w.write_event(Event::Empty(elem))?; + Ok(Visit::Continue) + } + + fn eof(&mut self) -> Result { + Ok(Visit::Continue) + } +} + +//------------------------------------------ From ed7480e96db32225693cd2dec9bba416f1c0dbd7 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 17 Sep 2021 17:43:15 +0800 Subject: [PATCH 06/25] [era_restore (rust)] First code drop --- src/bin/pdata_tools.rs | 2 + src/commands/era_restore.rs | 64 ++++++++ src/commands/mod.rs | 1 + src/era/mod.rs | 1 + src/era/restore.rs | 284 ++++++++++++++++++++++++++++++++++++ src/era/xml.rs | 150 ++++++++++++++++++- 6 files changed, 499 insertions(+), 3 deletions(-) create mode 100644 src/commands/era_restore.rs create mode 100644 src/era/restore.rs diff --git a/src/bin/pdata_tools.rs b/src/bin/pdata_tools.rs index 5081ac0..c2ababc 100644 --- a/src/bin/pdata_tools.rs +++ b/src/bin/pdata_tools.rs @@ -38,6 +38,8 @@ fn main_() -> Result<()> { era_check::run(&new_args); } else if name_eq(name, "era_dump") { era_dump::run(&new_args); + } else if name_eq(name, "era_restore") { + era_restore::run(&new_args); } else if name_eq(name, "thin_check") { thin_check::run(&new_args); } else if name_eq(name, "thin_dump") { diff --git a/src/commands/era_restore.rs b/src/commands/era_restore.rs new file mode 100644 index 0000000..0f1a542 --- /dev/null +++ b/src/commands/era_restore.rs @@ -0,0 +1,64 @@ +extern crate clap; + +use clap::{App, Arg}; +use std::path::Path; +use std::process; + +use crate::commands::utils::*; +use crate::era::restore::{restore, EraRestoreOptions}; + +pub fn run(args: &[std::ffi::OsString]) { + let parser = App::new("era_restore") + .version(crate::version::tools_version()) + .about("Convert XML format metadata to binary.") + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) + // options + .arg( + Arg::with_name("INPUT") + .help("Specify the input xml") + .short("i") + .long("input") + .value_name("FILE") + .required(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output device to check") + .short("o") + .long("output") + .value_name("FILE") + .required(true), + ); + + let matches = parser.get_matches_from(args); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); + + let report = mk_report(matches.is_present("QUIET")); + check_input_file(input_file, &report); + check_output_file(output_file, &report); + + let opts = EraRestoreOptions { + input: input_file, + output: output_file, + async_io: matches.is_present("ASYNC_IO"), + report: report.clone(), + }; + + if let Err(reason) = restore(opts) { + report.fatal(&format!("{}", reason)); + process::exit(1); + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index c070565..2abfeaa 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod cache_repair; pub mod cache_restore; pub mod era_check; pub mod era_dump; +pub mod era_restore; pub mod thin_check; pub mod thin_dump; pub mod thin_metadata_pack; diff --git a/src/era/mod.rs b/src/era/mod.rs index bb9a39f..9f6f3b4 100644 --- a/src/era/mod.rs +++ b/src/era/mod.rs @@ -1,6 +1,7 @@ pub mod check; pub mod dump; pub mod ir; +pub mod restore; pub mod superblock; pub mod writeset; pub mod xml; diff --git a/src/era/restore.rs b/src/era/restore.rs new file mode 100644 index 0000000..bbd05db --- /dev/null +++ b/src/era/restore.rs @@ -0,0 +1,284 @@ +use anyhow::{anyhow, Result}; + +use std::collections::BTreeMap; +use std::fs::OpenOptions; +use std::path::Path; +use std::sync::Arc; + +use crate::era::ir::{self, MetadataVisitor, Visit}; +use crate::era::superblock::*; +use crate::era::writeset::Writeset; +use crate::era::xml; +use crate::io_engine::*; +use crate::math::*; +use crate::pdata::array_builder::*; +use crate::pdata::btree_builder::*; +use crate::pdata::space_map_common::pack_root; +use crate::pdata::space_map_metadata::*; +use crate::report::*; +use crate::write_batcher::*; + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + +//------------------------------------------ + +pub struct EraRestoreOptions<'a> { + pub input: &'a Path, + pub output: &'a Path, + pub async_io: bool, + pub report: Arc, +} + +struct Context { + _report: Arc, + engine: Arc, +} + +fn mk_context(opts: &EraRestoreOptions) -> anyhow::Result { + let engine: Arc; + + if opts.async_io { + engine = Arc::new(AsyncIoEngine::new(opts.output, MAX_CONCURRENT_IO, true)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine = Arc::new(SyncIoEngine::new(opts.output, nr_threads, true)?); + } + + Ok(Context { + _report: opts.report.clone(), + engine, + }) +} + +//------------------------------------------ + +#[derive(PartialEq)] +enum Section { + None, + Superblock, + Writeset, + EraArray, + Finalized, +} + +pub struct Restorer<'a> { + w: &'a mut WriteBatcher, + sb: Option, + writesets: BTreeMap, + writeset_builder: Option>, // bitset + current_writeset: Option, + era_array_builder: Option>, + writeset_buf: (u32, u64), // (index in u64 array, value) + in_section: Section, +} + +impl<'a> Restorer<'a> { + pub fn new(w: &'a mut WriteBatcher) -> Restorer<'a> { + Restorer { + w, + sb: None, + writesets: BTreeMap::new(), + writeset_builder: None, + current_writeset: None, + era_array_builder: None, + writeset_buf: (0, 0), + in_section: Section::None, + } + } + + fn finalize(&mut self) -> Result<()> { + let src_sb; + if let Some(sb) = self.sb.take() { + src_sb = sb; + } else { + return Err(anyhow!("not in superblock")); + } + + // build the writeset tree + let mut tree_builder = BTreeBuilder::::new(Box::new(NoopRC {})); + let mut writesets = BTreeMap::::new(); + std::mem::swap(&mut self.writesets, &mut writesets); + for (era, ws) in writesets { + tree_builder.push_value(self.w, era as u64, ws)?; + } + let writeset_tree_root = tree_builder.complete(self.w)?; + + // complete the era array + let era_array_root; + if let Some(builder) = self.era_array_builder.take() { + era_array_root = builder.complete(self.w)?; + } else { + return Err(anyhow!("internal error. couldn't find era array")); + } + + // build metadata space map + let metadata_sm_root = build_metadata_sm(self.w)?; + + let sb = Superblock { + flags: SuperblockFlags { + clean_shutdown: true, + }, + block: SUPERBLOCK_LOCATION, + version: 1, + metadata_sm_root, + data_block_size: src_sb.block_size, + nr_blocks: src_sb.nr_blocks, + current_era: src_sb.current_era, + current_writeset: Writeset { + nr_bits: src_sb.nr_blocks, + root: 0, + }, + writeset_tree_root, + era_array_root, + metadata_snap: 0, + }; + write_superblock(self.w.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?; + + self.in_section = Section::Finalized; + Ok(()) + } +} + +impl<'a> MetadataVisitor for Restorer<'a> { + fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { + if self.in_section != Section::None { + return Err(anyhow!("duplicated superblock")); + } + + self.sb = Some(sb.clone()); + let b = self.w.alloc()?; + if b.loc != SUPERBLOCK_LOCATION { + return Err(anyhow!("superblock was occupied")); + } + + self.writeset_builder = None; + self.era_array_builder = Some(ArrayBuilder::new(sb.nr_blocks as u64)); + self.in_section = Section::Superblock; + + Ok(Visit::Continue) + } + + fn superblock_e(&mut self) -> Result { + self.finalize()?; + Ok(Visit::Continue) + } + + fn writeset_b(&mut self, ws: &ir::Writeset) -> Result { + if self.in_section != Section::Superblock { + return Err(anyhow!("not in superblock")); + } + self.writeset_builder = Some(ArrayBuilder::new(div_up(ws.nr_bits as u64, 64))); + self.writeset_buf.0 = 0; + self.writeset_buf.1 = 0; + self.current_writeset = Some(ws.clone()); + self.in_section = Section::Writeset; + Ok(Visit::Continue) + } + + fn writeset_e(&mut self) -> Result { + if self.in_section != Section::Writeset { + return Err(anyhow!("not in writeset")); + } + + if let Some(mut builder) = self.writeset_builder.take() { + if let Some(ws) = self.current_writeset.take() { + // push the trailing bits + builder.push_value(self.w, self.writeset_buf.0 as u64, self.writeset_buf.1)?; + + let root = builder.complete(self.w)?; + self.writesets.insert( + ws.era, + Writeset { + root, + nr_bits: ws.nr_bits, + }, + ); + self.in_section = Section::Superblock; + } else { + return Err(anyhow!("internal error. couldn't find writeset")); + } + } else { + return Err(anyhow!("internal error. couldn't find writeset")); + } + + Ok(Visit::Continue) + } + + fn writeset_bit(&mut self, wbit: &ir::WritesetBit) -> Result { + if wbit.value { + let index = wbit.block >> 6; + let mask = 1 << (wbit.block & 63); + if index == self.writeset_buf.0 { + self.writeset_buf.1 |= mask; + } else { + let builder = self.writeset_builder.as_mut().unwrap(); + builder.push_value(self.w, self.writeset_buf.0 as u64, self.writeset_buf.1)?; + self.writeset_buf.0 = index; + self.writeset_buf.1 = mask; + } + } + + Ok(Visit::Continue) + } + + fn era_b(&mut self) -> Result { + if self.in_section != Section::Superblock { + return Err(anyhow!("not in superblock")); + } + self.in_section = Section::EraArray; + Ok(Visit::Continue) + } + + fn era_e(&mut self) -> Result { + if self.in_section != Section::EraArray { + return Err(anyhow!("not in era array")); + } + self.in_section = Section::Superblock; + Ok(Visit::Continue) + } + + fn era(&mut self, era: &ir::Era) -> Result { + let builder = self.era_array_builder.as_mut().unwrap(); + builder.push_value(self.w, era.block as u64, era.era)?; + Ok(Visit::Continue) + } + + fn eof(&mut self) -> Result { + if self.in_section != Section::Finalized { + return Err(anyhow!("incompleted source metadata")); + } + Ok(Visit::Continue) + } +} + +//------------------------------------------ + +fn build_metadata_sm(w: &mut WriteBatcher) -> Result> { + let r = write_metadata_sm(w)?; + let sm_root = pack_root(&r, SPACE_MAP_ROOT_SIZE)?; + + Ok(sm_root) +} + +//------------------------------------------ + +pub fn restore(opts: EraRestoreOptions) -> Result<()> { + let input = OpenOptions::new() + .read(true) + .write(false) + .open(opts.input)?; + + let ctx = mk_context(&opts)?; + + let sm = core_metadata_sm(ctx.engine.get_nr_blocks(), u32::MAX); + let mut w = WriteBatcher::new(ctx.engine.clone(), sm.clone(), ctx.engine.get_batch_size()); + + let mut restorer = Restorer::new(&mut w); + xml::read(input, &mut restorer)?; + + Ok(()) +} + +//------------------------------------------ diff --git a/src/era/xml.rs b/src/era/xml.rs index 0b2dee7..4153a1f 100644 --- a/src/era/xml.rs +++ b/src/era/xml.rs @@ -1,7 +1,8 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use quick_xml::events::{BytesEnd, BytesStart, Event}; -use quick_xml::Writer; -use std::io::Write; +use quick_xml::{Reader, Writer}; +use std::io::{BufRead, BufReader}; +use std::io::{Read, Write}; use crate::era::ir::*; use crate::xml::*; @@ -91,3 +92,146 @@ impl MetadataVisitor for XmlWriter { } //------------------------------------------ + +fn parse_superblock(e: &BytesStart) -> Result { + let tag = "superblock"; + let mut uuid: Option = None; + let mut block_size: Option = None; + let mut nr_blocks: Option = None; + let mut current_era: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"uuid" => uuid = Some(string_val(&kv)), + b"block_size" => block_size = Some(u32_val(&kv)?), + b"nr_blocks" => nr_blocks = Some(u32_val(&kv)?), + b"current_era" => current_era = Some(u32_val(&kv)?), + _ => return bad_attr(tag, kv.key), + } + } + + Ok(Superblock { + uuid: check_attr(tag, "uuid", uuid)?, + block_size: check_attr(tag, "block_size", block_size)?, + nr_blocks: check_attr(tag, "nr_cache_blocks", nr_blocks)?, + current_era: check_attr(tag, "current_era", current_era)?, + }) +} + +fn parse_writeset(e: &BytesStart) -> Result { + let tag = "writeset"; + let mut era: Option = None; + let mut nr_bits: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"era" => era = Some(u32_val(&kv)?), + b"nr_bits" => nr_bits = Some(u32_val(&kv)?), + _ => return bad_attr(tag, kv.key), + } + } + + Ok(Writeset { + era: check_attr(tag, "era", era)?, + nr_bits: check_attr(tag, "nr_bits", nr_bits)?, + }) +} + +fn parse_writeset_bit(e: &BytesStart) -> Result { + let tag = "bit"; + let mut block: Option = None; + let mut value: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"block" => block = Some(u32_val(&kv)?), + b"value" => value = Some(bool_val(&kv)?), + _ => return bad_attr(tag, kv.key), + } + } + + Ok(WritesetBit { + block: check_attr(tag, "block", block)?, + value: check_attr(tag, "value", value)?, + }) +} + +fn parse_era(e: &BytesStart) -> Result { + let tag = "era"; + let mut block: Option = None; + let mut era: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"block" => block = Some(u32_val(&kv)?), + b"era" => era = Some(u32_val(&kv)?), + _ => return bad_attr(tag, kv.key), + } + } + + Ok(Era { + block: check_attr(tag, "block", block)?, + era: check_attr(tag, "era", era)?, + }) +} + +fn handle_event(reader: &mut Reader, buf: &mut Vec, visitor: &mut M) -> Result +where + R: Read + BufRead, + M: MetadataVisitor, +{ + match reader.read_event(buf) { + Ok(Event::Start(ref e)) => match e.name() { + b"superblock" => visitor.superblock_b(&parse_superblock(e)?), + b"writeset" => visitor.writeset_b(&parse_writeset(e)?), + b"era_array" => visitor.era_b(), + _ => return Err(anyhow!("Parse error at byte {}", reader.buffer_position())), + }, + Ok(Event::End(ref e)) => match e.name() { + b"superblock" => visitor.superblock_e(), + b"writeset" => visitor.writeset_e(), + b"era_array" => visitor.era_e(), + _ => return Err(anyhow!("Parse error at byte {}", reader.buffer_position())), + }, + Ok(Event::Empty(ref e)) => match e.name() { + b"bit" => visitor.writeset_bit(&parse_writeset_bit(e)?), + b"era" => visitor.era(&parse_era(e)?), + _ => return Err(anyhow!("Parse error at byte {}", reader.buffer_position())), + }, + Ok(Event::Text(_)) => Ok(Visit::Continue), + Ok(Event::Comment(_)) => Ok(Visit::Continue), + Ok(Event::Eof) => { + visitor.eof()?; + Ok(Visit::Stop) + } + Ok(_) => return Err(anyhow!("Parse error at byte {}", reader.buffer_position())), + Err(e) => { + return Err(anyhow!( + "Parse error at byte {}: {:?}", + reader.buffer_position(), + e + )) + } + } +} + +pub fn read(input: R, visitor: &mut M) -> Result<()> +where + R: Read, + M: MetadataVisitor, +{ + let input = BufReader::new(input); + let mut reader = Reader::from_reader(input); + + reader.trim_text(true); + let mut buf = Vec::new(); + + while let Visit::Continue = handle_event(&mut reader, &mut buf, visitor)? {} + Ok(()) +} + +//------------------------------------------ From 3a8dc8da2dc0efebdddb0c9de6d7edbfec4106d8 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 17 Sep 2021 22:00:43 +0800 Subject: [PATCH 07/25] [xml (rust)] Log invalid attribute name --- src/xml.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/xml.rs b/src/xml.rs index 2b617bf..722d70c 100644 --- a/src/xml.rs +++ b/src/xml.rs @@ -32,8 +32,14 @@ pub fn bool_val(kv: &Attribute) -> anyhow::Result { Ok(n) } -pub fn bad_attr(tag: &str, _attr: &[u8]) -> anyhow::Result { - Err(anyhow!("unknown attribute in tag '{}'", tag)) +pub fn bad_attr(tag: &str, attr: &[u8]) -> anyhow::Result { + Err(anyhow!( + "unknown attribute {}in tag '{}'", + std::str::from_utf8(attr) + .map(|s| format!("'{}' ", s)) + .unwrap_or_default(), + tag + )) } pub fn check_attr(tag: &str, name: &str, maybe_v: Option) -> anyhow::Result { From 36767bcda6c59802e089d106a2cc17d941a95a3c Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 22 Sep 2021 16:58:04 +0800 Subject: [PATCH 08/25] [era_repair (rust)] First code drop --- src/commands/era_repair.rs | 73 ++++++++++++++++++++++++++++++++++++++ src/commands/mod.rs | 1 + src/era/mod.rs | 1 + src/era/repair.rs | 68 +++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 src/commands/era_repair.rs create mode 100644 src/era/repair.rs diff --git a/src/commands/era_repair.rs b/src/commands/era_repair.rs new file mode 100644 index 0000000..6f05764 --- /dev/null +++ b/src/commands/era_repair.rs @@ -0,0 +1,73 @@ +extern crate clap; + +use atty::Stream; +use clap::{App, Arg}; +use std::path::Path; +use std::process; +use std::sync::Arc; + +use crate::commands::utils::*; +use crate::era::repair::{repair, EraRepairOptions}; +use crate::report::*; + +pub fn run(args: &[std::ffi::OsString]) { + let parser = App::new("era_repair") + .version(crate::version::tools_version()) + .about("Repair binary era metadata, and write it to a different device or file") + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) + // options + .arg( + Arg::with_name("INPUT") + .help("Specify the input device") + .short("i") + .long("input") + .value_name("FILE") + .required(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output device") + .short("o") + .long("output") + .value_name("FILE") + .required(true), + ); + + let matches = parser.get_matches_from(args); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); + + let report = if matches.is_present("QUIET") { + std::sync::Arc::new(mk_quiet_report()) + } else if atty::is(Stream::Stdout) { + std::sync::Arc::new(mk_progress_bar_report()) + } else { + Arc::new(mk_simple_report()) + }; + + check_input_file(input_file, &report); + + let opts = EraRepairOptions { + input: input_file, + output: output_file, + async_io: matches.is_present("ASYNC_IO"), + report: report.clone(), + }; + + if let Err(reason) = repair(opts) { + report.fatal(&format!("{}", reason)); + process::exit(1); + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 2abfeaa..c1d35e8 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod cache_repair; pub mod cache_restore; pub mod era_check; pub mod era_dump; +pub mod era_repair; pub mod era_restore; pub mod thin_check; pub mod thin_dump; diff --git a/src/era/mod.rs b/src/era/mod.rs index 9f6f3b4..da8383a 100644 --- a/src/era/mod.rs +++ b/src/era/mod.rs @@ -1,6 +1,7 @@ pub mod check; pub mod dump; pub mod ir; +pub mod repair; pub mod restore; pub mod superblock; pub mod writeset; diff --git a/src/era/repair.rs b/src/era/repair.rs new file mode 100644 index 0000000..2139a8e --- /dev/null +++ b/src/era/repair.rs @@ -0,0 +1,68 @@ +use anyhow::Result; +use std::path::Path; +use std::sync::Arc; + +use crate::era::dump::*; +use crate::era::restore::*; +use crate::era::superblock::*; +use crate::io_engine::*; +use crate::pdata::space_map_metadata::*; +use crate::report::*; +use crate::write_batcher::*; + +//------------------------------------------ + +pub struct EraRepairOptions<'a> { + pub input: &'a Path, + pub output: &'a Path, + pub async_io: bool, + pub report: Arc, +} + +struct Context { + _report: Arc, + engine_in: Arc, + engine_out: Arc, +} + +const MAX_CONCURRENT_IO: u32 = 1024; + +fn new_context(opts: &EraRepairOptions) -> Result { + let engine_in: Arc; + let engine_out: Arc; + + if opts.async_io { + engine_in = Arc::new(AsyncIoEngine::new(opts.input, MAX_CONCURRENT_IO, false)?); + engine_out = Arc::new(AsyncIoEngine::new(opts.output, MAX_CONCURRENT_IO, true)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine_in = Arc::new(SyncIoEngine::new(opts.input, nr_threads, false)?); + engine_out = Arc::new(SyncIoEngine::new(opts.output, nr_threads, true)?); + } + + Ok(Context { + _report: opts.report.clone(), + engine_in, + engine_out, + }) +} + +//------------------------------------------ + +pub fn repair(opts: EraRepairOptions) -> Result<()> { + let ctx = new_context(&opts)?; + + let sb = read_superblock(ctx.engine_in.as_ref(), SUPERBLOCK_LOCATION)?; + + let sm = core_metadata_sm(ctx.engine_out.get_nr_blocks(), u32::MAX); + let mut w = WriteBatcher::new( + ctx.engine_out.clone(), + sm.clone(), + ctx.engine_out.get_batch_size(), + ); + let mut restorer = Restorer::new(&mut w); + + dump_metadata(ctx.engine_in, &mut restorer, &sb, true) +} + +//------------------------------------------ From 9ab417279035502d6ca001e6a5833659a9078d42 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 23 Sep 2021 16:59:54 +0800 Subject: [PATCH 09/25] [cache_restore (rust)] Tidy up --- src/cache/restore.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cache/restore.rs b/src/cache/restore.rs index 07970c7..ba2db51 100644 --- a/src/cache/restore.rs +++ b/src/cache/restore.rs @@ -218,9 +218,9 @@ impl<'a> MetadataVisitor for Restorer<'a> { if m.dirty { let index = m.cblock >> 6; - let bi = m.cblock & 63; + let mask = 1 << (m.cblock & 63); if index == self.dirty_bits.0 { - self.dirty_bits.1 |= 1 << bi; + self.dirty_bits.1 |= mask; } else { let dirty_builder = self.dirty_builder.as_mut().unwrap(); dirty_builder.push_value( From 0791208ca4bfa4f67bf6b55c1203f5d659a68a18 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 24 Sep 2021 18:24:13 +0800 Subject: [PATCH 10/25] [io_engine (rust)] Open file exclusively --- src/io_engine.rs | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/io_engine.rs b/src/io_engine.rs index 55782dc..091ed76 100644 --- a/src/io_engine.rs +++ b/src/io_engine.rs @@ -131,21 +131,30 @@ impl<'a> Drop for FileGuard<'a> { } impl SyncIoEngine { - fn open_file(path: &Path, writable: bool) -> Result { + fn open_file(path: &Path, writable: bool, excl: bool) -> Result { let file = OpenOptions::new() .read(true) .write(writable) - .custom_flags(libc::O_EXCL) + .custom_flags(if excl { libc::O_EXCL } else { 0 }) .open(path)?; Ok(file) } pub fn new(path: &Path, nr_files: usize, writable: bool) -> Result { + SyncIoEngine::new_with(path, nr_files, writable, true) + } + + pub fn new_with( + path: &Path, + nr_files: usize, + writable: bool, + excl: bool, + ) -> Result { let nr_blocks = get_nr_blocks(path)?; // check file mode eariler let mut files = Vec::with_capacity(nr_files); for _n in 0..nr_files { - files.push(SyncIoEngine::open_file(path, writable)?); + files.push(SyncIoEngine::open_file(path, writable, excl)?); } Ok(SyncIoEngine { @@ -237,11 +246,24 @@ pub struct AsyncIoEngine { impl AsyncIoEngine { pub fn new(path: &Path, queue_len: u32, writable: bool) -> Result { + AsyncIoEngine::new_with(path, queue_len, writable, true) + } + + pub fn new_with( + path: &Path, + queue_len: u32, + writable: bool, + excl: bool, + ) -> Result { let nr_blocks = get_nr_blocks(path)?; // check file mode earlier + let mut flags = libc::O_DIRECT; + if excl { + flags |= libc::O_EXCL; + } let input = OpenOptions::new() .read(true) .write(writable) - .custom_flags(libc::O_DIRECT | libc::O_EXCL) + .custom_flags(flags) .open(path)?; Ok(AsyncIoEngine { From 4559039066fe71df0af8951b04285ff0376a32e8 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 25 Sep 2021 02:04:38 +0800 Subject: [PATCH 11/25] [era_invalidate (rust)] First code drop --- src/commands/era_invalidate.rs | 85 ++++++++++ src/commands/mod.rs | 1 + src/era/invalidate.rs | 285 +++++++++++++++++++++++++++++++++ src/era/mod.rs | 1 + 4 files changed, 372 insertions(+) create mode 100644 src/commands/era_invalidate.rs create mode 100644 src/era/invalidate.rs diff --git a/src/commands/era_invalidate.rs b/src/commands/era_invalidate.rs new file mode 100644 index 0000000..c877ba4 --- /dev/null +++ b/src/commands/era_invalidate.rs @@ -0,0 +1,85 @@ +extern crate clap; + +use clap::{App, Arg}; +use std::path::Path; +use std::process; + +use crate::commands::utils::*; +use crate::era::invalidate::{invalidate, EraInvalidateOptions}; + +//------------------------------------------ + +pub fn run(args: &[std::ffi::OsString]) { + let parser = App::new("era_invalidate") + .version(crate::version::tools_version()) + .about("List blocks that may have changed since a given era") + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + // options + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output file rather than stdout") + .short("o") + .long("output") + .value_name("FILE"), + ) + // arguments + .arg( + Arg::with_name("INPUT") + .help("Specify the input device to dump") + .required(true) + .index(1), + ) + .arg( + Arg::with_name("WRITTEN_SINCE") + .help("Blocks written since the given era will be listed") + .long("written-since") + .required(true) + .value_name("ERA"), + ); + + let matches = parser.get_matches_from(args); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = if matches.is_present("OUTPUT") { + Some(Path::new(matches.value_of("OUTPUT").unwrap())) + } else { + None + }; + + // Create a temporary report just in case these checks + // need to report anything. + let report = std::sync::Arc::new(crate::report::mk_simple_report()); + check_input_file(input_file, &report); + check_file_not_tiny(input_file, &report); + drop(report); + + let threshold = matches + .value_of("WRITTEN_SINCE") + .map(|s| { + s.parse::().unwrap_or_else(|_| { + eprintln!("Couldn't parse written_since"); + process::exit(1); + }) + }) + .unwrap_or(0); + + let opts = EraInvalidateOptions { + input: input_file, + output: output_file, + async_io: matches.is_present("ASYNC_IO"), + threshold, + metadata_snap: matches.is_present("METADATA_SNAP"), + }; + + if let Err(reason) = invalidate(&opts) { + eprintln!("{}", reason); + process::exit(1); + } +} + +//------------------------------------------ diff --git a/src/commands/mod.rs b/src/commands/mod.rs index c1d35e8..5dd1f19 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod cache_repair; pub mod cache_restore; pub mod era_check; pub mod era_dump; +pub mod era_invalidate; pub mod era_repair; pub mod era_restore; pub mod thin_check; diff --git a/src/era/invalidate.rs b/src/era/invalidate.rs new file mode 100644 index 0000000..ab9ccc4 --- /dev/null +++ b/src/era/invalidate.rs @@ -0,0 +1,285 @@ +use anyhow::Result; +use quick_xml::events::{BytesEnd, BytesStart, Event}; +use quick_xml::Writer; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use crate::era::superblock::*; +use crate::era::writeset::*; +use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; +use crate::math::div_up; +use crate::pdata::array::{self, value_err, ArrayBlock}; +use crate::pdata::array_walker::*; +use crate::pdata::btree_walker::*; +use crate::xml::mk_attr; + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + +//------------------------------------------ + +struct BitsetCollator<'a> { + composed_bits: Box>, +} + +impl<'a> BitsetCollator<'a> { + fn new(bitset: &mut [u64]) -> BitsetCollator { + BitsetCollator { + composed_bits: Box::new(Mutex::new(bitset)), + } + } +} + +impl<'a> ArrayVisitor for BitsetCollator<'a> { + fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { + let mut bitset = self.composed_bits.lock().unwrap(); + let idx = index as usize * b.header.max_entries as usize; // index of u64 in bitset array + for (entry, dest) in b.values.iter().zip(bitset.iter_mut().skip(idx)) { + *dest |= entry; + } + Ok(()) + } +} + +//------------------------------------------ + +struct EraArrayCollator<'a> { + composed_bits: Box>, + threshold: u32, +} + +impl<'a> EraArrayCollator<'a> { + fn new(bitset: &mut [u64], threshold: u32) -> EraArrayCollator { + EraArrayCollator { + composed_bits: Box::new(Mutex::new(bitset)), + threshold, + } + } +} + +impl<'a> ArrayVisitor for EraArrayCollator<'a> { + fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { + let blk_begin = index as usize * b.header.max_entries as usize; // range of data blocks + let blk_end = blk_begin + b.header.max_entries as usize; + + let mut bitset = self.composed_bits.lock().unwrap(); + let mut bitset_iter = bitset.iter_mut(); + let mut idx = blk_begin >> 6; // index of u64 in bitset array + let mut dest = bitset_iter + .nth(idx) + .ok_or_else(|| value_err("array index out of bounds".to_string()))?; + let mut buf = *dest; + for (era, blk) in b.values.iter().zip(blk_begin..blk_end) { + if *era < self.threshold { + continue; + } + let steps = (blk >> 6) - idx; + if steps > 0 { + *dest = buf; + idx += steps; + dest = bitset_iter + .nth(steps - 1) + .ok_or_else(|| value_err("array index out of bounds".to_string()))?; + buf = *dest; + } + buf |= 1 << (blk & 0x3F); + } + *dest = buf; + + Ok(()) + } +} + +//------------------------------------------ + +fn collate_writeset( + engine: Arc, + writeset_root: u64, + marked_bits: &mut [u64], +) -> Result<()> { + let w = ArrayWalker::new(engine, false); + let mut c = BitsetCollator::new(marked_bits); + w.walk(&mut c, writeset_root)?; + Ok(()) +} + +fn collate_era_array( + engine: Arc, + era_array_root: u64, + marked_bits: &mut [u64], + threshold: u32, +) -> Result<()> { + let w = ArrayWalker::new(engine, false); + let mut c = EraArrayCollator::new(marked_bits, threshold); + w.walk(&mut c, era_array_root)?; + Ok(()) +} + +fn mark_blocks_since( + engine: Arc, + sb: &Superblock, + threshold: u32, +) -> Result> { + let mut marked_bits = Vec::::new(); + marked_bits.resize(div_up(sb.nr_blocks as usize, 64), 0); + + let mut path = vec![0]; + let wsets = btree_to_map::(&mut path, engine.clone(), false, sb.writeset_tree_root)?; + for (era, ws) in wsets.iter() { + if (*era as u32) < threshold { + continue; + } + collate_writeset(engine.clone(), ws.root, &mut marked_bits)?; + } + + if let Some(archived_begin) = wsets.keys().next() { + if *archived_begin as u32 > threshold { + collate_era_array( + engine.clone(), + sb.era_array_root, + &mut marked_bits, + threshold, + )?; + } + } + + Ok(marked_bits) +} + +fn emit_start(w: &mut Writer) -> Result<()> { + let elem = BytesStart::owned_name(b"blocks".to_vec()); + w.write_event(Event::Start(elem))?; + Ok(()) +} + +fn emit_end(w: &mut Writer) -> Result<()> { + let elem = BytesEnd::borrowed(b"blocks"); + w.write_event(Event::End(elem))?; + Ok(()) +} + +fn emit_range(w: &mut Writer, begin: u32, end: u32) -> Result<()> { + if end > begin + 1 { + let mut elem = BytesStart::owned_name(b"range".to_vec()); + elem.push_attribute(mk_attr(b"begin", begin)); + elem.push_attribute(mk_attr(b"end", end)); + w.write_event(Event::Empty(elem))?; + } else if end > begin { + let mut elem = BytesStart::owned_name(b"block".to_vec()); + elem.push_attribute(mk_attr(b"block", begin)); + w.write_event(Event::Empty(elem))?; + } + + Ok(()) +} + +fn emit_blocks(marked_bits: &[u64], nr_blocks: u32, w: &mut Writer) -> Result<()> { + let mut begin: u32 = 0; + let mut end: u32 = 0; + + emit_start(w)?; + + for (index, entry) in marked_bits.iter().enumerate() { + let mut n = *entry; + + if n == u64::max_value() { + end = std::cmp::min(end + 64, nr_blocks); + continue; + } + + while n > 0 { + let zeros = n.trailing_zeros(); + if zeros > 0 { + if end > begin { + emit_range(w, begin, end)?; + } + n >>= zeros; + end += zeros; + begin = end; + } + + let ones = n.trailing_ones(); + n >>= ones; + end = std::cmp::min(end + ones, nr_blocks); + } + + let endpos = (index << 6) as u32 + 64; + if end < endpos { + if end > begin { + emit_range(w, begin, end)?; + } + begin = endpos; + end = begin; + } + } + + if end > begin { + emit_range(w, begin, end)?; + } + + emit_end(w)?; + + Ok(()) +} + +//------------------------------------------ + +pub struct EraInvalidateOptions<'a> { + pub input: &'a Path, + pub output: Option<&'a Path>, + pub async_io: bool, + pub threshold: u32, + pub metadata_snap: bool, +} + +struct Context { + engine: Arc, +} + +fn mk_context(opts: &EraInvalidateOptions) -> anyhow::Result { + let engine: Arc; + + if opts.async_io { + engine = Arc::new(AsyncIoEngine::new_with( + opts.input, + MAX_CONCURRENT_IO, + false, + !opts.metadata_snap, + )?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine = Arc::new(SyncIoEngine::new_with( + opts.input, + nr_threads, + false, + !opts.metadata_snap, + )?); + } + + Ok(Context { engine }) +} + +pub fn invalidate(opts: &EraInvalidateOptions) -> Result<()> { + let ctx = mk_context(opts)?; + + let mut sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; + if opts.metadata_snap { + sb = read_superblock(ctx.engine.as_ref(), sb.metadata_snap)?; + } + + let w: Box; + if opts.output.is_some() { + w = Box::new(BufWriter::new(File::create(opts.output.unwrap())?)); + } else { + w = Box::new(BufWriter::new(std::io::stdout())); + } + let mut writer = Writer::new_with_indent(w, 0x20, 2); + + let marked_bits = mark_blocks_since(ctx.engine, &sb, opts.threshold)?; + emit_blocks(&marked_bits, sb.nr_blocks, &mut writer) +} + +//------------------------------------------ diff --git a/src/era/mod.rs b/src/era/mod.rs index da8383a..7f5cd93 100644 --- a/src/era/mod.rs +++ b/src/era/mod.rs @@ -1,5 +1,6 @@ pub mod check; pub mod dump; +pub mod invalidate; pub mod ir; pub mod repair; pub mod restore; From 8fa7f96dfdcc49e136b5b18e6cefd18d4c877773 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 29 Sep 2021 16:57:04 +0800 Subject: [PATCH 12/25] [era_dump/restore (rust)] Support displaying writeset blocks in ranges - Group adjacent marked blocks into ranges - Show marked blocks only, and ignore untouched blocks --- src/era/dump.rs | 75 ++++++++++++++++++++++++-------- src/era/ir.rs | 8 ++-- src/era/restore.rs | 63 +++++++++++++++++++-------- src/era/xml.rs | 102 ++++++++++++++++++++++++++++++++++++++------ src/pdata/bitset.rs | 66 ++++++++++++++++++++++++++++ 5 files changed, 263 insertions(+), 51 deletions(-) diff --git a/src/era/dump.rs b/src/era/dump.rs index a72bd51..cc3a9b1 100644 --- a/src/era/dump.rs +++ b/src/era/dump.rs @@ -1,4 +1,3 @@ -use anyhow::anyhow; use std::fs::File; use std::io::BufWriter; use std::io::Write; @@ -12,7 +11,7 @@ use crate::era::xml; use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::*; -use crate::pdata::bitset::read_bitset; +use crate::pdata::bitset::read_bitset_no_err; use crate::pdata::btree_walker::btree_to_map; //------------------------------------------ @@ -78,6 +77,7 @@ fn mk_context(opts: &EraDumpOptions) -> anyhow::Result { Ok(Context { engine }) } +// notify the visitor about the marked blocks only fn dump_writeset( engine: Arc, out: &mut dyn MetadataVisitor, @@ -85,27 +85,68 @@ fn dump_writeset( ws: &Writeset, repair: bool, ) -> anyhow::Result<()> { - let (bits, errs) = read_bitset(engine.clone(), ws.root, ws.nr_bits as usize, repair); // TODO: deal with broken writeset - if errs.is_some() { - return Err(anyhow!( - "errors in writeset of era {}: {}", - era, - errs.unwrap() - )); - } + let bits = read_bitset_no_err(engine.clone(), ws.root, ws.nr_bits as usize, repair)?; out.writeset_b(&ir::Writeset { era, nr_bits: ws.nr_bits, })?; - for b in 0..ws.nr_bits { - let wbit = ir::WritesetBit { - block: b, - value: bits.contains(b as usize).unwrap_or(false), - }; - out.writeset_bit(&wbit)?; + + // [begin, end) denotes the range of set bits. + let mut begin: u32 = 0; + let mut end: u32 = 0; + for (index, entry) in bits.as_slice().iter().enumerate() { + let mut n = *entry; + + if n == u32::MAX { + end = std::cmp::min(end + 32, ws.nr_bits); + continue; + } + + while n > 0 { + let zeros = n.trailing_zeros(); + if zeros > 0 { + if end > begin { + let m = ir::MarkedBlocks { + begin, + len: end - begin, + }; + out.writeset_blocks(&m)?; + } + n >>= zeros; + end += zeros; + begin = end; + } + + let ones = n.trailing_ones(); + n >>= ones; + end = std::cmp::min(end + ones, ws.nr_bits); + } + + // emit the range if it ends before the entry boundary + let endpos = ((index as u32) << 5) + 32; + if end < endpos { + if end > begin { + let m = ir::MarkedBlocks { + begin, + len: end - begin, + }; + out.writeset_blocks(&m)?; + } + begin = endpos; + end = begin; + } } + + if end > begin { + let m = ir::MarkedBlocks { + begin, + len: end - begin, + }; + out.writeset_blocks(&m)?; + } + out.writeset_e()?; Ok(()) @@ -164,7 +205,7 @@ pub fn dump(opts: EraDumpOptions) -> anyhow::Result<()> { } else { writer = Box::new(BufWriter::new(std::io::stdout())); } - let mut out = xml::XmlWriter::new(writer); + let mut out = xml::XmlWriter::new(writer, false); if opts.logical { dump_metadata_logical(ctx.engine, &mut out, &sb, opts.repair) diff --git a/src/era/ir.rs b/src/era/ir.rs index 0dc1342..947a4f6 100644 --- a/src/era/ir.rs +++ b/src/era/ir.rs @@ -17,9 +17,9 @@ pub struct Writeset { } #[derive(Clone)] -pub struct WritesetBit { - pub block: u32, - pub value: bool, +pub struct MarkedBlocks { + pub begin: u32, + pub len: u32, } #[derive(Clone)] @@ -42,7 +42,7 @@ pub trait MetadataVisitor { fn writeset_b(&mut self, ws: &Writeset) -> Result; fn writeset_e(&mut self) -> Result; - fn writeset_bit(&mut self, wbit: &WritesetBit) -> Result; + fn writeset_blocks(&mut self, blocks: &MarkedBlocks) -> Result; fn era_b(&mut self) -> Result; fn era_e(&mut self) -> Result; diff --git a/src/era/restore.rs b/src/era/restore.rs index bbd05db..127abca 100644 --- a/src/era/restore.rs +++ b/src/era/restore.rs @@ -70,7 +70,8 @@ pub struct Restorer<'a> { writeset_builder: Option>, // bitset current_writeset: Option, era_array_builder: Option>, - writeset_buf: (u32, u64), // (index in u64 array, value) + writeset_entry: u64, + entry_index: u32, in_section: Section, } @@ -83,7 +84,8 @@ impl<'a> Restorer<'a> { writeset_builder: None, current_writeset: None, era_array_builder: None, - writeset_buf: (0, 0), + writeset_entry: 0, + entry_index: 0, in_section: Section::None, } } @@ -170,8 +172,8 @@ impl<'a> MetadataVisitor for Restorer<'a> { return Err(anyhow!("not in superblock")); } self.writeset_builder = Some(ArrayBuilder::new(div_up(ws.nr_bits as u64, 64))); - self.writeset_buf.0 = 0; - self.writeset_buf.1 = 0; + self.entry_index = 0; + self.writeset_entry = 0; self.current_writeset = Some(ws.clone()); self.in_section = Section::Writeset; Ok(Visit::Continue) @@ -185,7 +187,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { if let Some(mut builder) = self.writeset_builder.take() { if let Some(ws) = self.current_writeset.take() { // push the trailing bits - builder.push_value(self.w, self.writeset_buf.0 as u64, self.writeset_buf.1)?; + builder.push_value(self.w, self.entry_index as u64, self.writeset_entry)?; let root = builder.complete(self.w)?; self.writesets.insert( @@ -206,20 +208,47 @@ impl<'a> MetadataVisitor for Restorer<'a> { Ok(Visit::Continue) } - fn writeset_bit(&mut self, wbit: &ir::WritesetBit) -> Result { - if wbit.value { - let index = wbit.block >> 6; - let mask = 1 << (wbit.block & 63); - if index == self.writeset_buf.0 { - self.writeset_buf.1 |= mask; - } else { - let builder = self.writeset_builder.as_mut().unwrap(); - builder.push_value(self.w, self.writeset_buf.0 as u64, self.writeset_buf.1)?; - self.writeset_buf.0 = index; - self.writeset_buf.1 = mask; - } + fn writeset_blocks(&mut self, blocks: &ir::MarkedBlocks) -> Result { + let first = blocks.begin; + let last = first + blocks.len - 1; // inclusive + let mut idx = first >> 6; + let last_idx = last >> 6; // inclusive + let builder = self.writeset_builder.as_mut().unwrap(); + + // emit the bufferred bits + if idx > self.entry_index { + builder.push_value(self.w, self.entry_index as u64, self.writeset_entry)?; + self.entry_index = idx; + self.writeset_entry = 0; } + // buffer the bits of the first entry + let bi_first = first & 63; + if idx == last_idx { + let bi_last = last & 63; + let mask = 1u64 << bi_last; + self.writeset_entry |= (mask ^ mask.wrapping_sub(1)) & (u64::MAX << bi_first); + + return Ok(Visit::Continue); + } + + self.writeset_entry |= u64::MAX << bi_first; + + // emit the all-1 entries if necessary + while idx < last_idx { + builder.push_value(self.w, self.entry_index as u64, self.writeset_entry)?; + self.entry_index += 1; + self.writeset_entry = u64::MAX; + idx += 1; + } + + // buffer the bits of the last entry + builder.push_value(self.w, self.entry_index as u64, self.writeset_entry)?; + let bi_last = last & 63; + let mask = 1u64 << bi_last; + self.entry_index += 1; + self.writeset_entry |= mask ^ mask.wrapping_sub(1); + Ok(Visit::Continue) } diff --git a/src/era/xml.rs b/src/era/xml.rs index 4153a1f..7b54b79 100644 --- a/src/era/xml.rs +++ b/src/era/xml.rs @@ -11,12 +11,18 @@ use crate::xml::*; pub struct XmlWriter { w: Writer, + compact: bool, + nr_blocks: u32, + emitted_blocks: u32, } impl XmlWriter { - pub fn new(w: W) -> XmlWriter { + pub fn new(w: W, compact: bool) -> XmlWriter { XmlWriter { w: Writer::new_with_indent(w, 0x20, 2), + compact, + nr_blocks: 0, + emitted_blocks: 0, } } } @@ -29,8 +35,10 @@ impl MetadataVisitor for XmlWriter { elem.push_attribute(mk_attr(b"block_size", sb.block_size)); elem.push_attribute(mk_attr(b"nr_blocks", sb.nr_blocks)); elem.push_attribute(mk_attr(b"current_era", sb.current_era)); - self.w.write_event(Event::Start(elem))?; + + self.nr_blocks = sb.nr_blocks; + Ok(Visit::Continue) } @@ -46,21 +54,55 @@ impl MetadataVisitor for XmlWriter { elem.push_attribute(mk_attr(b"era", ws.era)); elem.push_attribute(mk_attr(b"nr_bits", ws.nr_bits)); self.w.write_event(Event::Start(elem))?; + + self.emitted_blocks = 0; + Ok(Visit::Continue) } fn writeset_e(&mut self) -> Result { + if !self.compact { + for b in self.emitted_blocks..self.nr_blocks { + let tag = b"bit"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"block", b)); + elem.push_attribute(mk_attr(b"value", "false")); + self.w.write_event(Event::Empty(elem))?; + } + } self.w .write_event(Event::End(BytesEnd::borrowed(b"writeset")))?; Ok(Visit::Continue) } - fn writeset_bit(&mut self, wbit: &WritesetBit) -> Result { - let tag = b"bit"; - let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); - elem.push_attribute(mk_attr(b"block", wbit.block)); - elem.push_attribute(mk_attr(b"value", wbit.value)); - self.w.write_event(Event::Empty(elem))?; + fn writeset_blocks(&mut self, blocks: &MarkedBlocks) -> Result { + if self.compact { + let tag = b"marked"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"block_begin", blocks.begin)); + elem.push_attribute(mk_attr(b"len", blocks.len)); + self.w.write_event(Event::Empty(elem))?; + } else { + for b in self.emitted_blocks..blocks.begin { + let tag = b"bit"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"block", b)); + elem.push_attribute(mk_attr(b"value", "false")); + self.w.write_event(Event::Empty(elem))?; + } + + let end = blocks.begin + blocks.len; + for b in blocks.begin..end { + let tag = b"bit"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"block", b)); + elem.push_attribute(mk_attr(b"value", "true")); + self.w.write_event(Event::Empty(elem))?; + } + + self.emitted_blocks = end; + } + Ok(Visit::Continue) } @@ -139,7 +181,7 @@ fn parse_writeset(e: &BytesStart) -> Result { }) } -fn parse_writeset_bit(e: &BytesStart) -> Result { +fn parse_writeset_bit(e: &BytesStart) -> Result> { let tag = "bit"; let mut block: Option = None; let mut value: Option = None; @@ -153,9 +195,36 @@ fn parse_writeset_bit(e: &BytesStart) -> Result { } } - Ok(WritesetBit { - block: check_attr(tag, "block", block)?, - value: check_attr(tag, "value", value)?, + check_attr(tag, "block", block)?; + check_attr(tag, "value", value)?; + + if let Some(true) = value { + Ok(Some(MarkedBlocks { + begin: block.unwrap(), + len: 1, + })) + } else { + Ok(None) + } +} + +fn parse_writeset_blocks(e: &BytesStart) -> Result { + let tag = "marked"; + let mut begin: Option = None; + let mut len: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"block_begin" => begin = Some(u32_val(&kv)?), + b"len" => len = Some(u32_val(&kv)?), + _ => return bad_attr(tag, kv.key), + } + } + + Ok(MarkedBlocks { + begin: check_attr(tag, "block_begin", begin)?, + len: check_attr(tag, "len", len)?, }) } @@ -198,7 +267,14 @@ where _ => return Err(anyhow!("Parse error at byte {}", reader.buffer_position())), }, Ok(Event::Empty(ref e)) => match e.name() { - b"bit" => visitor.writeset_bit(&parse_writeset_bit(e)?), + b"bit" => { + if let Some(b) = parse_writeset_bit(e)? { + visitor.writeset_blocks(&b) + } else { + Ok(Visit::Continue) + } + } + b"marked" => visitor.writeset_blocks(&parse_writeset_blocks(e)?), b"era" => visitor.era(&parse_era(e)?), _ => return Err(anyhow!("Parse error at byte {}", reader.buffer_position())), }, diff --git a/src/pdata/bitset.rs b/src/pdata/bitset.rs index fcd0942..86352b9 100644 --- a/src/pdata/bitset.rs +++ b/src/pdata/bitset.rs @@ -2,10 +2,13 @@ use fixedbitset::FixedBitSet; use std::sync::{Arc, Mutex}; use crate::io_engine::IoEngine; +use crate::math::div_up; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker}; use crate::pdata::space_map::*; +//------------------------------------------ + pub struct CheckedBitSet { bits: FixedBitSet, } @@ -30,6 +33,8 @@ impl CheckedBitSet { } } +//------------------------------------------ + struct BitsetVisitor { nr_bits: usize, bits: Mutex, @@ -72,6 +77,55 @@ impl ArrayVisitor for BitsetVisitor { } } +//------------------------------------------ + +struct BitsetCollector { + bits: Mutex, + nr_bits: usize, +} + +impl BitsetCollector { + fn new(nr_bits: usize) -> BitsetCollector { + BitsetCollector { + bits: Mutex::new(FixedBitSet::with_capacity(nr_bits)), + nr_bits, + } + } + + pub fn get_bitset(self) -> FixedBitSet { + self.bits.into_inner().unwrap() + } +} + +impl ArrayVisitor for BitsetCollector { + fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { + let mut bitset = self.bits.lock().unwrap(); + let mut idx = (index as usize * b.header.max_entries as usize) << 1; // index of u32 in bitset array + let idx_end = div_up(self.nr_bits, 32); + let mut dest = bitset.as_mut_slice().iter_mut().skip(idx); + for entry in b.values.iter() { + let lower = (*entry & (u32::MAX as u64)) as u32; + *(dest.next().ok_or_else(|| { + array::value_err(format!("bitset size exceeds limit: {} bits", self.nr_bits)) + })?) = lower; + idx += 1; + + if idx == idx_end { + break; + } + + let upper = (*entry >> 32) as u32; + *(dest.next().ok_or_else(|| { + array::value_err(format!("bitset size exceeds limit: {} bits", self.nr_bits)) + })?) = upper; + idx += 1; + } + Ok(()) + } +} + +//------------------------------------------ + // TODO: multi-threaded is possible pub fn read_bitset( engine: Arc, @@ -106,3 +160,15 @@ pub fn read_bitset_with_sm( }; Ok((v.get_bitset(), e)) } + +pub fn read_bitset_no_err( + engine: Arc, + root: u64, + nr_bits: usize, + ignore_none_fatal: bool, +) -> array::Result { + let w = ArrayWalker::new(engine, ignore_none_fatal); + let mut v = BitsetCollector::new(nr_bits); + w.walk(&mut v, root)?; + Ok(v.get_bitset()) +} From 0215b5fecdb9d71d9a8eaef1012e208841f174c0 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 29 Sep 2021 17:57:59 +0800 Subject: [PATCH 13/25] [era_check (rust)] Remove unused option --- src/commands/era_check.rs | 6 ------ src/era/check.rs | 1 - 2 files changed, 7 deletions(-) diff --git a/src/commands/era_check.rs b/src/commands/era_check.rs index 34c1ce8..3467e8e 100644 --- a/src/commands/era_check.rs +++ b/src/commands/era_check.rs @@ -22,11 +22,6 @@ pub fn run(args: &[std::ffi::OsString]) { .long("async-io") .hidden(true), ) - .arg( - Arg::with_name("AUTO_REPAIR") - .help("Auto repair trivial issues.") - .long("auto-repair"), - ) .arg( Arg::with_name("IGNORE_NON_FATAL") .help("Only return a non-zero exit code if a fatal error is found.") @@ -71,7 +66,6 @@ pub fn run(args: &[std::ffi::OsString]) { async_io: matches.is_present("ASYNC_IO"), sb_only: matches.is_present("SB_ONLY"), ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"), - auto_repair: matches.is_present("AUTO_REPAIR"), report: report.clone(), }; diff --git a/src/era/check.rs b/src/era/check.rs index ab23ad0..dbde8d2 100644 --- a/src/era/check.rs +++ b/src/era/check.rs @@ -66,7 +66,6 @@ pub struct EraCheckOptions<'a> { pub async_io: bool, pub sb_only: bool, pub ignore_non_fatal: bool, - pub auto_repair: bool, pub report: Arc, } From 321fce882fa62ccf6937bdca702fb237af7ea4bf Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 29 Sep 2021 21:06:44 +0800 Subject: [PATCH 14/25] [tests] Move era_check tests to Rust --- functional-tests/era-functional-tests.scm | 77 -------------- tests/common/target.rs | 32 ++++++ tests/era_check.rs | 118 ++++++++++++++++++++++ 3 files changed, 150 insertions(+), 77 deletions(-) create mode 100644 tests/era_check.rs diff --git a/functional-tests/era-functional-tests.scm b/functional-tests/era-functional-tests.scm index a81b41b..f15c599 100644 --- a/functional-tests/era-functional-tests.scm +++ b/functional-tests/era-functional-tests.scm @@ -43,83 +43,6 @@ (define (register-era-tests) #t) - ;;;----------------------------------------------------------- - ;;; era_check scenarios - ;;;----------------------------------------------------------- - (define-scenario (era-check v) - "era_check -V" - (run-ok-rcv (stdout _) (era-check "-V") - (assert-equal tools-version stdout))) - - (define-scenario (era-check version) - "era_check --version" - (run-ok-rcv (stdout _) (era-check "--version") - (assert-equal tools-version stdout))) - - (define-scenario (era-check h) - "era_check -h" - (run-ok-rcv (stdout _) (era-check "-h") - (assert-equal era-check-help stdout))) - - (define-scenario (era-check help) - "era_check --help" - (run-ok-rcv (stdout _) (era-check "--help") - (assert-equal era-check-help stdout))) - - (define-scenario (era-check no-device-specified) - "Fail if no device specified" - (run-fail-rcv (_ stderr) (era-check) - (assert-starts-with "No input file provided." stderr))) - - (define-scenario (era-check dev-not-exist) - "Fail if specified device doesn't exist" - (run-fail-rcv (_ stderr) (era-check "/dev/unlikely") - (assert-starts-with "/dev/unlikely: No such file or directory" stderr))) - - (define-scenario (era-check dev-is-a-directory) - "Fail if given a directory instead of a file or device" - (run-fail-rcv (_ stderr) (era-check "/tmp") - (assert-starts-with "/tmp: Not a block device or regular file" stderr))) - - (define-scenario (era-check bad-permissions) - "Fail if given a device with inadequate access permissions" - (with-temp-file-sized ((md "era.bin" (meg 4))) - (run-ok "chmod -r" md) - (run-fail-rcv (_ stderr) (era-check md) - (assert-starts-with "syscall 'open' failed: Permission denied" stderr)))) - - (define-scenario (era-check empty-dev) - "Fail if given a file of zeroes" - (with-empty-metadata (md) - (run-fail (era-check md)))) - - (define-scenario (era-check quiet) - "Fail should give no output if --quiet" - (with-empty-metadata (md) - (run-fail-rcv (stdout stderr) (era-check "--quiet" md) - (assert-eof stdout) - (assert-eof stderr)))) - - (define-scenario (era-check q) - "Fail should give no output if -q" - (with-empty-metadata (md) - (run-fail-rcv (stdout stderr) (era-check "-q" md) - (assert-eof stdout) - (assert-eof stderr)))) - - (define-scenario (era-check tiny-metadata) - "Prints helpful message in case tiny metadata given" - (with-temp-file-sized ((md "era.bin" 1024)) - (run-fail-rcv (_ stderr) (era-check md) - (assert-starts-with "Metadata device/file too small. Is this binary metadata?" stderr)))) - - (define-scenario (era-check spot-accidental-xml-data) - "Prints helpful message if XML metadata given" - (with-era-xml (xml) - (system (fmt #f "man bash >> " xml)) - (run-fail-rcv (_ stderr) (era-check xml) - (assert-matches ".*This looks like XML. era_check only checks the binary metadata format." stderr)))) - ;;;----------------------------------------------------------- ;;; era_restore scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/target.rs b/tests/common/target.rs index 6a91930..fa8ffc0 100644 --- a/tests/common/target.rs +++ b/tests/common/target.rs @@ -159,6 +159,38 @@ where rust_cmd("cache_repair", args) } +pub fn era_check_cmd(args: I) -> Command +where + I: IntoIterator, + I::Item: Into, +{ + rust_cmd("era_check", args) +} + +pub fn era_dump_cmd(args: I) -> Command +where + I: IntoIterator, + I::Item: Into, +{ + rust_cmd("era_dump", args) +} + +pub fn era_restore_cmd(args: I) -> Command +where + I: IntoIterator, + I::Item: Into, +{ + rust_cmd("era_restore", args) +} + +pub fn era_repair_cmd(args: I) -> Command +where + I: IntoIterator, + I::Item: Into, +{ + rust_cmd("era_repair", args) +} + //------------------------------------------ pub mod msg { diff --git a/tests/era_check.rs b/tests/era_check.rs new file mode 100644 index 0000000..774991f --- /dev/null +++ b/tests/era_check.rs @@ -0,0 +1,118 @@ +use anyhow::Result; + +mod common; + +use common::cache::*; +use common::common_args::*; +use common::fixture::*; +use common::input_arg::*; +use common::process::*; +use common::program::*; +use common::target::*; +use common::test_dir::*; + +//------------------------------------------ + +const USAGE: &str = "era_check 0.9.0 + +USAGE: + era_check [FLAGS] + +FLAGS: + --ignore-non-fatal-errors Only return a non-zero exit code if a fatal error is found. + -q, --quiet Suppress output messages, return only exit code. + --super-block-only Only check the superblock. + -h, --help Prints help information + -V, --version Prints version information + +ARGS: + Specify the input device to check"; + +//------------------------------------------ + +struct EraCheck; + +impl<'a> Program<'a> for EraCheck { + fn name() -> &'a str { + "era_check" + } + + fn cmd(args: I) -> Command + where + I: IntoIterator, + I::Item: Into, + { + era_check_cmd(args) + } + + fn usage() -> &'a str { + USAGE + } + + fn arg_type() -> ArgType { + ArgType::InputArg + } + + fn bad_option_hint(option: &str) -> String { + msg::bad_option_hint(option) + } +} + +impl<'a> InputProgram<'a> for EraCheck { + fn mk_valid_input(td: &mut TestDir) -> Result { + mk_valid_md(td) + } + + fn file_not_found() -> &'a str { + msg::FILE_NOT_FOUND + } + + fn missing_input_arg() -> &'a str { + msg::MISSING_INPUT_ARG + } + + fn corrupted_input() -> &'a str { + msg::BAD_SUPERBLOCK + } +} + +impl<'a> MetadataReader<'a> for EraCheck {} + +//------------------------------------------ + +test_accepts_help!(EraCheck); +test_accepts_version!(EraCheck); +test_rejects_bad_option!(EraCheck); + +test_missing_input_arg!(EraCheck); +test_input_file_not_found!(EraCheck); +test_input_cannot_be_a_directory!(EraCheck); +test_unreadable_input_file!(EraCheck); + +test_help_message_for_tiny_input_file!(EraCheck); +test_spot_xml_data!(EraCheck); +test_corrupted_input_data!(EraCheck); + +//------------------------------------------ + +#[test] +fn failing_q() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let output = run_fail_raw(era_check_cmd(args!["-q", &md]))?; + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +#[test] +fn failing_quiet() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let output = run_fail_raw(era_check_cmd(args!["--quiet", &md]))?; + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +//------------------------------------------ From 4cfe93570cdb3950cf1a2351210f6da4a60ffaf4 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 1 Oct 2021 15:49:23 +0800 Subject: [PATCH 15/25] [tests] Add era test fixtures --- tests/common/era.rs | 34 +++++++ tests/common/era_xml_generator.rs | 157 ++++++++++++++++++++++++++++++ tests/common/mod.rs | 2 + 3 files changed, 193 insertions(+) create mode 100644 tests/common/era.rs create mode 100644 tests/common/era_xml_generator.rs diff --git a/tests/common/era.rs b/tests/common/era.rs new file mode 100644 index 0000000..c7b66a0 --- /dev/null +++ b/tests/common/era.rs @@ -0,0 +1,34 @@ +use anyhow::Result; +use std::path::PathBuf; + +use thinp::file_utils; + +use crate::args; +use crate::common::era_xml_generator::{write_xml, CleanShutdownMeta}; +use crate::common::process::*; +use crate::common::target::*; +use crate::common::test_dir::TestDir; + +//----------------------------------------------- + +pub fn mk_valid_xml(td: &mut TestDir) -> Result { + let xml = td.mk_path("meta.xml"); + let mut gen = CleanShutdownMeta::new(128, 256, 32, 4); // bs, nr_blocks, era, nr_wsets + write_xml(&xml, &mut gen)?; + Ok(xml) +} + +pub fn mk_valid_md(td: &mut TestDir) -> Result { + let xml = td.mk_path("meta.xml"); + let md = td.mk_path("meta.bin"); + + let mut gen = CleanShutdownMeta::new(128, 256, 32, 4); + write_xml(&xml, &mut gen)?; + + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + run_ok(ERA_RESTORE, args!["-i", &xml, "-o", &md])?; + + Ok(md) +} + +//----------------------------------------------- diff --git a/tests/common/era_xml_generator.rs b/tests/common/era_xml_generator.rs new file mode 100644 index 0000000..8c7fad4 --- /dev/null +++ b/tests/common/era_xml_generator.rs @@ -0,0 +1,157 @@ +use anyhow::Result; +use rand::prelude::*; +use std::fs::OpenOptions; +use std::path::Path; +use thinp::era::ir::{self, MetadataVisitor}; +use thinp::era::xml; + +//------------------------------------------ + +pub trait XmlGen { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()>; +} + +pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { + let xml_out = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + let mut w = xml::XmlWriter::new(xml_out, false); + + g.generate_xml(&mut w) +} + +//------------------------------------------ + +// Ordered sequence generator where each element has an independent probability +// of being present. +struct IndependentSequence { + begin: u32, + end: u32, + prob: u32, + rng: ThreadRng, +} + +impl IndependentSequence { + fn new(begin: u32, end: u32, prob: u32) -> IndependentSequence { + IndependentSequence { + begin, + end, + prob, + rng: rand::thread_rng(), + } + } +} + +impl Iterator for IndependentSequence { + type Item = std::ops::Range; + + // FIXME: reduce complexity + fn next(&mut self) -> Option> { + if self.begin >= self.end { + return None; + } + + let mut b = self.begin; + while b < self.end && self.rng.gen_range(0..100) >= self.prob { + b += 1; + } + + if b == self.end { + return None; + } + + let mut e = b + 1; + while e < self.end && self.rng.gen_range(0..100) < self.prob { + e += 1; + } + self.begin = e + 1; + + Some(std::ops::Range { start: b, end: e }) + } +} + +//------------------------------------------ + +fn create_superblock(block_size: u32, nr_blocks: u32, current_era: u32) -> ir::Superblock { + ir::Superblock { + uuid: "".to_string(), + block_size, + nr_blocks, + current_era, + } +} + +pub struct CleanShutdownMeta { + block_size: u32, + nr_blocks: u32, + current_era: u32, + nr_writesets: u32, +} + +impl CleanShutdownMeta { + pub fn new(block_size: u32, nr_blocks: u32, current_era: u32, nr_writesets: u32) -> Self { + CleanShutdownMeta { + block_size, + nr_blocks, + current_era, + nr_writesets, + } + } + + fn generate_writeset(v: &mut dyn MetadataVisitor, ws: &ir::Writeset) -> Result<()> { + v.writeset_b(ws)?; + let gen = IndependentSequence::new(0, ws.nr_bits, 10); + for seq in gen { + v.writeset_blocks(&ir::MarkedBlocks { + begin: seq.start, + len: seq.end - seq.start, + })?; + } + v.writeset_e()?; + + Ok(()) + } + + fn generate_era_array(v: &mut dyn MetadataVisitor, nr_blocks: u32, max_era: u32) -> Result<()> { + let mut rng = rand::thread_rng(); + v.era_b()?; + for b in 0..nr_blocks { + let era = rng.gen_range(0..max_era); + v.era(&ir::Era { block: b, era })?; + } + v.era_e()?; + + Ok(()) + } +} + +impl XmlGen for CleanShutdownMeta { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> { + v.superblock_b(&create_superblock( + self.block_size, + self.nr_blocks, + self.current_era, + ))?; + + let era_low = self.current_era - self.nr_writesets + 1; + for era in era_low..self.current_era + 1 { + Self::generate_writeset( + v, + &ir::Writeset { + era, + nr_bits: self.nr_blocks, + }, + )?; + } + + Self::generate_era_array(v, self.nr_blocks, era_low)?; + + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 93f6db4..e434e72 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -5,6 +5,8 @@ pub mod cache; pub mod cache_xml_generator; pub mod common_args; +pub mod era; +pub mod era_xml_generator; pub mod fixture; pub mod input_arg; pub mod output_option; From 8d3f65244d61d7a1c93ecac97f3fe0a065d37c98 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 1 Oct 2021 23:31:32 +0800 Subject: [PATCH 16/25] [tests] Move era_restore tests to Rust --- functional-tests/era-functional-tests.scm | 70 ----------- tests/common/era.rs | 2 +- tests/era_restore.rs | 143 ++++++++++++++++++++++ 3 files changed, 144 insertions(+), 71 deletions(-) create mode 100644 tests/era_restore.rs diff --git a/functional-tests/era-functional-tests.scm b/functional-tests/era-functional-tests.scm index f15c599..3847b82 100644 --- a/functional-tests/era-functional-tests.scm +++ b/functional-tests/era-functional-tests.scm @@ -46,76 +46,6 @@ ;;;----------------------------------------------------------- ;;; era_restore scenarios ;;;----------------------------------------------------------- - (define-scenario (era-restore v) - "era_restore -V" - (run-ok-rcv (stdout _) (era-restore "-V") - (assert-equal tools-version stdout))) - - (define-scenario (era-restore version) - "era_restore --version" - (run-ok-rcv (stdout _) (era-restore "--version") - (assert-equal tools-version stdout))) - - (define-scenario (era-restore h) - "era_restore -h" - (run-ok-rcv (stdout _) (era-restore "-h") - (assert-equal era-restore-help stdout))) - - (define-scenario (era-restore help) - "era_restore --help" - (run-ok-rcv (stdout _) (era-restore "--help") - (assert-equal era-restore-help stdout))) - - (define-scenario (era-restore input-unspecified) - "Fails if no xml specified" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (era-restore "-o" md) - (assert-starts-with "No input file provided." stderr)))) - - (define-scenario (era-restore missing-input-file) - "the input file can't be found" - (with-empty-metadata (md) - (let ((bad-path "no-such-file")) - (run-fail-rcv (_ stderr) (era-restore "-i no-such-file -o" md) - (assert-superblock-all-zeroes md) - (assert-starts-with - (string-append bad-path ": No such file or directory") - stderr))))) - - (define-scenario (era-restore garbage-input-file) - "the input file is just zeroes" - (with-empty-metadata (md) - (with-temp-file-sized ((xml "era.xml" 4096)) - (run-fail-rcv (_ stderr) (era-restore "-i " xml "-o" md) - (assert-superblock-all-zeroes md))))) - - (define-scenario (era-restore output-unspecified) - "Fails if no metadata dev specified" - (with-era-xml (xml) - (run-fail-rcv (_ stderr) (era-restore "-i" xml) - (assert-starts-with "No output file provided." stderr)))) - - (define-scenario (era-restore success) - "Succeeds with xml and metadata" - (with-era-xml (xml) - (with-empty-metadata (md) - (run-ok (era-restore "-i" xml "-o" md))))) - - (define-scenario (era-restore quiet) - "No output with --quiet (succeeding)" - (with-era-xml (xml) - (with-empty-metadata (md) - (run-ok-rcv (stdout stderr) (era-restore "--quiet" "-i" xml "-o" md) - (assert-eof stdout) - (assert-eof stderr))))) - - (define-scenario (era-restore q) - "No output with -q (succeeding)" - (with-era-xml (xml) - (with-empty-metadata (md) - (run-ok-rcv (stdout stderr) (era-restore "-q" "-i" xml "-o" md) - (assert-eof stdout) - (assert-eof stderr))))) (define-scenario (era-restore quiet-fail) "No output with --quiet (failing)" diff --git a/tests/common/era.rs b/tests/common/era.rs index c7b66a0..12406dc 100644 --- a/tests/common/era.rs +++ b/tests/common/era.rs @@ -26,7 +26,7 @@ pub fn mk_valid_md(td: &mut TestDir) -> Result { write_xml(&xml, &mut gen)?; let _file = file_utils::create_sized_file(&md, 4096 * 4096); - run_ok(ERA_RESTORE, args!["-i", &xml, "-o", &md])?; + run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md]))?; Ok(md) } diff --git a/tests/era_restore.rs b/tests/era_restore.rs new file mode 100644 index 0000000..eadbe10 --- /dev/null +++ b/tests/era_restore.rs @@ -0,0 +1,143 @@ +use anyhow::Result; + +mod common; + +use common::common_args::*; +use common::era::*; +use common::fixture::*; +use common::input_arg::*; +use common::output_option::*; +use common::process::*; +use common::program::*; +use common::target::*; +use common::test_dir::*; + +//------------------------------------------ + +const USAGE: &str = "era_restore 0.9.0 +Convert XML format metadata to binary. + +USAGE: + era_restore [FLAGS] --input --output + +FLAGS: + -q, --quiet Suppress output messages, return only exit code. + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + -i, --input Specify the input xml + -o, --output Specify the output device to check"; + +//------------------------------------------ + +struct EraRestore; + +impl<'a> Program<'a> for EraRestore { + fn name() -> &'a str { + "era_restore" + } + + fn cmd(args: I) -> Command + where + I: IntoIterator, + I::Item: Into, + { + era_restore_cmd(args) + } + + fn usage() -> &'a str { + USAGE + } + + fn arg_type() -> ArgType { + ArgType::IoOptions + } + + fn bad_option_hint(option: &str) -> String { + msg::bad_option_hint(option) + } +} + +impl<'a> InputProgram<'a> for EraRestore { + fn mk_valid_input(td: &mut TestDir) -> Result { + mk_valid_xml(td) + } + + fn file_not_found() -> &'a str { + msg::FILE_NOT_FOUND + } + + fn missing_input_arg() -> &'a str { + msg::MISSING_INPUT_ARG + } + + fn corrupted_input() -> &'a str { + "" // we don't intent to verify error messages of XML parsing + } +} + +impl<'a> OutputProgram<'a> for EraRestore { + fn missing_output_arg() -> &'a str { + msg::MISSING_OUTPUT_ARG + } +} + +impl<'a> MetadataWriter<'a> for EraRestore { + fn file_not_found() -> &'a str { + msg::FILE_NOT_FOUND + } +} + +//----------------------------------------- + +test_accepts_help!(EraRestore); +test_accepts_version!(EraRestore); + +test_missing_input_option!(EraRestore); +test_input_file_not_found!(EraRestore); +test_corrupted_input_data!(EraRestore); + +test_missing_output_option!(EraRestore); +test_tiny_output_file!(EraRestore); + +test_unwritable_output_file!(EraRestore); + +//----------------------------------------- + +// TODO: share with thin_restore, era_restore + +fn quiet_flag(flag: &str) -> Result<()> { + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; + let md = mk_zeroed_md(&mut td)?; + + let output = run_ok_raw(era_restore_cmd(args!["-i", &xml, "-o", &md, flag]))?; + + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +#[test] +fn accepts_q() -> Result<()> { + quiet_flag("-q") +} + +#[test] +fn accepts_quiet() -> Result<()> { + quiet_flag("--quiet") +} + +//----------------------------------------- + +#[test] +fn successfully_restores() -> Result<()> { + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; + let md = mk_zeroed_md(&mut td)?; + run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md]))?; + Ok(()) +} + +//----------------------------------------- From f6eb5173c9cd20ce1506b3173b313675e9d6820d Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 2 Oct 2021 01:26:29 +0800 Subject: [PATCH 17/25] [era_restore] Avoid touching the output file by checking input file earlier The output file has been checked by the caller, so there's no need to check the output file again. --- era/era_restore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/era/era_restore.cc b/era/era_restore.cc index 27dcadc..a99f763 100644 --- a/era/era_restore.cc +++ b/era/era_restore.cc @@ -37,7 +37,7 @@ namespace { bool metadata_touched = false; try { block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE); - file_utils::check_file_exists(*fs.output, false); + file_utils::check_file_exists(*fs.input); metadata_touched = true; metadata::ptr md(new metadata(bm, metadata::CREATE)); emitter::ptr restorer = create_restore_emitter(*md); From cbc3baba4589bdfdb65048671c814175163c113e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 5 Oct 2021 15:22:23 +0800 Subject: [PATCH 18/25] [tests] Move era_dump tests to Rust --- functional-tests/era-functional-tests.scm | 17 --- tests/common/input_arg.rs | 26 +++++ tests/era_dump.rs | 123 ++++++++++++++++++++++ 3 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 tests/era_dump.rs diff --git a/functional-tests/era-functional-tests.scm b/functional-tests/era-functional-tests.scm index 3847b82..7a24f6b 100644 --- a/functional-tests/era-functional-tests.scm +++ b/functional-tests/era-functional-tests.scm @@ -66,21 +66,4 @@ (assert-starts-with (string-append bad-xml ": No such file or directory") stderr))))) - - ;;;----------------------------------------------------------- - ;;; era_dump scenarios - ;;;----------------------------------------------------------- - (define-scenario (era-dump small-input-file) - "Fails with small input file" - (with-temp-file-sized ((md "era.bin" 512)) - (run-fail (era-dump md)))) - - (define-scenario (era-dump restore-is-noop) - "era_dump followed by era_restore is a noop." - (with-valid-metadata (md) - (run-ok-rcv (d1-stdout _) (era-dump md) - (with-temp-file-containing ((xml "era.xml" d1-stdout)) - (run-ok (era-restore "-i" xml "-o" md)) - (run-ok-rcv (d2-stdout _) (era-dump md) - (assert-equal d1-stdout d2-stdout)))))) ) diff --git a/tests/common/input_arg.rs b/tests/common/input_arg.rs index 2dfae10..e1970f8 100644 --- a/tests/common/input_arg.rs +++ b/tests/common/input_arg.rs @@ -180,6 +180,32 @@ macro_rules! test_unreadable_input_file { //------------------------------------------ // test invalid content +pub fn test_tiny_input_file<'a, P>() -> Result<()> +where + P: MetadataReader<'a>, +{ + let mut td = TestDir::new()?; + + let input = td.mk_path("meta.bin"); + file_utils::create_sized_file(&input, 1024)?; + + let wrapper = build_args_fn(P::arg_type())?; + wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| { + run_fail(P::cmd(args))?; + Ok(()) + }) +} + +#[macro_export] +macro_rules! test_tiny_input_file { + ($program: ident) => { + #[test] + fn tiny_input_file() -> Result<()> { + test_tiny_input_file::<$program>() + } + }; +} + pub fn test_help_message_for_tiny_input_file<'a, P>() -> Result<()> where P: MetadataReader<'a>, diff --git a/tests/era_dump.rs b/tests/era_dump.rs new file mode 100644 index 0000000..8e71108 --- /dev/null +++ b/tests/era_dump.rs @@ -0,0 +1,123 @@ +use anyhow::Result; +use std::fs::OpenOptions; +use std::io::Write; + +mod common; + +use common::common_args::*; +use common::era::*; +use common::fixture::*; +use common::input_arg::*; +use common::process::*; +use common::program::*; +use common::target::*; +use common::test_dir::*; + +//------------------------------------------ + +const USAGE: &str = "era_dump 0.9.0 +Dump the era metadata to stdout in XML format + +USAGE: + era_dump [FLAGS] [OPTIONS] + +FLAGS: + --logical Fold any unprocessed write sets into the final era array + -r, --repair Repair the metadata whilst dumping it + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + -o, --output Specify the output file rather than stdout + +ARGS: + Specify the input device to dump"; + +//------------------------------------------ + +struct EraDump; + +impl<'a> Program<'a> for EraDump { + fn name() -> &'a str { + "era_dump" + } + + fn cmd(args: I) -> Command + where + I: IntoIterator, + I::Item: Into, + { + era_dump_cmd(args) + } + + fn usage() -> &'a str { + USAGE + } + + fn arg_type() -> ArgType { + ArgType::InputArg + } + + fn bad_option_hint(option: &str) -> String { + msg::bad_option_hint(option) + } +} + +impl<'a> InputProgram<'a> for EraDump { + fn mk_valid_input(td: &mut TestDir) -> Result { + mk_valid_md(td) + } + + fn file_not_found() -> &'a str { + msg::FILE_NOT_FOUND + } + + fn missing_input_arg() -> &'a str { + msg::MISSING_INPUT_ARG + } + + fn corrupted_input() -> &'a str { + msg::BAD_SUPERBLOCK + } +} + +impl<'a> MetadataReader<'a> for EraDump {} + +//------------------------------------------ + +test_accepts_help!(EraDump); +test_accepts_version!(EraDump); +test_rejects_bad_option!(EraDump); + +test_missing_input_arg!(EraDump); +test_input_file_not_found!(EraDump); +test_input_cannot_be_a_directory!(EraDump); +test_unreadable_input_file!(EraDump); +test_tiny_input_file!(EraDump); + +//------------------------------------------ + +// TODO: share with thin_dump +#[test] +fn dump_restore_cycle() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let output = run_ok_raw(era_dump_cmd(args![&md]))?; + + let xml = td.mk_path("meta.xml"); + let mut file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .open(&xml)?; + file.write_all(&output.stdout[0..])?; + drop(file); + + let md2 = mk_zeroed_md(&mut td)?; + run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md2]))?; + + let output2 = run_ok_raw(era_dump_cmd(args![&md2]))?; + assert_eq!(output.stdout, output2.stdout); + + Ok(()) +} From 89e568b8976ba109a9d35a88ab1411324acd8c68 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 6 Oct 2021 23:08:47 +0800 Subject: [PATCH 19/25] [era_dump (rust)] Implement logical dump --- src/era/dump.rs | 196 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 187 insertions(+), 9 deletions(-) diff --git a/src/era/dump.rs b/src/era/dump.rs index cc3a9b1..531b45b 100644 --- a/src/era/dump.rs +++ b/src/era/dump.rs @@ -1,6 +1,10 @@ +use anyhow::Result; +use fixedbitset::FixedBitSet; +use std::convert::TryFrom; use std::fs::File; use std::io::BufWriter; use std::io::Write; +use std::ops::Deref; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -12,7 +16,7 @@ use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::*; use crate::pdata::bitset::read_bitset_no_err; -use crate::pdata::btree_walker::btree_to_map; +use crate::pdata::btree_walker::{btree_to_key_set, btree_to_map}; //------------------------------------------ @@ -52,6 +56,112 @@ impl<'a> ArrayVisitor for EraEmitter<'a> { //------------------------------------------ +trait Archive { + fn set(&mut self, key: u32, value: u32) -> Result<()>; + fn get(&self, key: u32) -> Result; +} + +// in-core archive of writeset eras +struct EraArchive { + digested_era: u32, // maximum possible era in the era array + deltas: Vec, +} + +fn new_era_archive(nr_blocks: u32, archived_min: u32, nr_writesets: u32) -> Box { + match nr_writesets + 1 { + 0..=255 => Box::new(EraArchive { + digested_era: archived_min.wrapping_sub(1), + deltas: vec![0u8; nr_blocks as usize], + }), + 256..=65535 => Box::new(EraArchive { + digested_era: archived_min.wrapping_sub(1), + deltas: vec![0u16; nr_blocks as usize], + }), + _ => Box::new(EraArchive { + digested_era: archived_min.wrapping_sub(1), + deltas: vec![0u32; nr_blocks as usize], + }), + } +} + +impl> Archive for EraArchive +where + T: Copy + Into + TryFrom, + >::Error: std::fmt::Debug, +{ + fn set(&mut self, block: u32, delta: u32) -> Result<()> { + self.deltas[block as usize] = T::try_from(delta).unwrap(); + Ok(()) + } + + fn get(&self, block: u32) -> Result { + let delta: u32 = self.deltas[block as usize].into(); + if delta > 0 { + Ok(self.digested_era.wrapping_add(delta)) + } else { + Ok(0) + } + } +} + +//------------------------------------------ + +struct Inner<'a> { + emitter: &'a mut dyn MetadataVisitor, + era_archive: &'a dyn Archive, +} + +struct LogicalEraEmitter<'a> { + inner: Mutex>, +} + +impl<'a> LogicalEraEmitter<'a> { + pub fn new( + emitter: &'a mut dyn MetadataVisitor, + era_archive: &'a dyn Archive, + ) -> LogicalEraEmitter<'a> { + LogicalEraEmitter { + inner: Mutex::new(Inner { + emitter, + era_archive, + }), + } + } +} + +impl<'a> ArrayVisitor for LogicalEraEmitter<'a> { + fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { + let mut inner = self.inner.lock().unwrap(); + + let begin = index as u32 * b.header.max_entries; + let end = begin + b.header.nr_entries; + for (v, block) in b.values.iter().zip(begin..end) { + let archived = inner + .era_archive + .get(block) + .map_err(|e| array::value_err(format!("{}", e)))?; + + let era = if archived > 0 { + ir::Era { + block, + era: archived, + } + } else { + ir::Era { block, era: *v } + }; + + inner + .emitter + .era(&era) + .map_err(|e| array::value_err(format!("{}", e)))?; + } + + Ok(()) + } +} + +//------------------------------------------ + pub struct EraDumpOptions<'a> { pub input: &'a Path, pub output: Option<&'a Path>, @@ -185,16 +295,76 @@ pub fn dump_metadata( Ok(()) } -pub fn dump_metadata_logical( - _engine: Arc, - _out: &mut dyn MetadataVisitor, - _sb: &Superblock, - _repair: bool, -) -> anyhow::Result<()> { - // TODO +//----------------------------------------- + +fn collate_writeset(index: u32, bitset: &FixedBitSet, archive: &mut dyn Archive) -> Result<()> { + let era_delta = index + 1; + + for (i, entry) in bitset.as_slice().iter().enumerate() { + let mut bi = (i << 5) as u32; + let mut n = *entry; + while n > 0 { + if n & 0x1 > 0 { + archive.set(bi, era_delta)?; + } + n >>= 1; + bi += 1; + } + } + Ok(()) } +fn collate_writesets( + engine: Arc, + sb: &Superblock, + repair: bool, +) -> Result> { + let mut path = vec![0]; + let writesets = + btree_to_map::(&mut path, engine.clone(), repair, sb.writeset_tree_root)?; + + let archived_min = writesets.iter().next().map_or(0u32, |(k, _v)| *k as u32); + let mut archive = new_era_archive(sb.nr_blocks, archived_min, writesets.len() as u32); + + for (index, (_era, ws)) in writesets.iter().enumerate() { + let bitset = read_bitset_no_err(engine.clone(), ws.root, ws.nr_bits as usize, repair)?; + collate_writeset(index as u32, &bitset, archive.as_mut())?; + } + + Ok(archive) +} + +pub fn dump_metadata_logical( + engine: Arc, + out: &mut dyn MetadataVisitor, + sb: &Superblock, + repair: bool, +) -> anyhow::Result<()> { + let era_archive = collate_writesets(engine.clone(), sb, repair)?; + + let xml_sb = ir::Superblock { + uuid: "".to_string(), + block_size: sb.data_block_size, + nr_blocks: sb.nr_blocks, + current_era: sb.current_era, + }; + out.superblock_b(&xml_sb)?; + + out.era_b()?; + let w = ArrayWalker::new(engine, repair); + let mut emitter = LogicalEraEmitter::new(out, era_archive.deref()); + w.walk(&mut emitter, sb.era_array_root)?; + out.era_e()?; + + out.superblock_e()?; + out.eof()?; + + Ok(()) +} + +//----------------------------------------- + pub fn dump(opts: EraDumpOptions) -> anyhow::Result<()> { let ctx = mk_context(&opts)?; let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; @@ -207,7 +377,15 @@ pub fn dump(opts: EraDumpOptions) -> anyhow::Result<()> { } let mut out = xml::XmlWriter::new(writer, false); - if opts.logical { + let mut path = vec![0]; + let writesets = btree_to_key_set::( + &mut path, + ctx.engine.clone(), + opts.repair, + sb.writeset_tree_root, + )?; + + if opts.logical && !writesets.is_empty() { dump_metadata_logical(ctx.engine, &mut out, &sb, opts.repair) } else { dump_metadata(ctx.engine, &mut out, &sb, opts.repair) From 533e174051d9b6dd582583e2eb5602e270d946a5 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 8 Oct 2021 10:22:20 +0800 Subject: [PATCH 20/25] [era_dump (rust)] Support wrapped-around era in logical dump --- src/era/dump.rs | 95 +++++++++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/src/era/dump.rs b/src/era/dump.rs index 531b45b..d81ffd2 100644 --- a/src/era/dump.rs +++ b/src/era/dump.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use std::convert::TryFrom; use std::fs::File; @@ -16,7 +16,7 @@ use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::*; use crate::pdata::bitset::read_bitset_no_err; -use crate::pdata::btree_walker::{btree_to_key_set, btree_to_map}; +use crate::pdata::btree_walker::btree_to_map; //------------------------------------------ @@ -58,27 +58,28 @@ impl<'a> ArrayVisitor for EraEmitter<'a> { trait Archive { fn set(&mut self, key: u32, value: u32) -> Result<()>; - fn get(&self, key: u32) -> Result; + fn get(&self, key: u32) -> Option; } -// in-core archive of writeset eras +// In-core archive of writeset eras. +// The actual era for a given block is `digested_era + deltas[b]` if `deltas[b]` is non-zero. struct EraArchive { digested_era: u32, // maximum possible era in the era array deltas: Vec, } -fn new_era_archive(nr_blocks: u32, archived_min: u32, nr_writesets: u32) -> Box { +fn new_era_archive(nr_blocks: u32, archived_begin: u32, nr_writesets: u32) -> Box { match nr_writesets + 1 { 0..=255 => Box::new(EraArchive { - digested_era: archived_min.wrapping_sub(1), + digested_era: archived_begin.wrapping_sub(1), deltas: vec![0u8; nr_blocks as usize], }), 256..=65535 => Box::new(EraArchive { - digested_era: archived_min.wrapping_sub(1), + digested_era: archived_begin.wrapping_sub(1), deltas: vec![0u16; nr_blocks as usize], }), _ => Box::new(EraArchive { - digested_era: archived_min.wrapping_sub(1), + digested_era: archived_begin.wrapping_sub(1), deltas: vec![0u32; nr_blocks as usize], }), } @@ -94,12 +95,16 @@ where Ok(()) } - fn get(&self, block: u32) -> Result { - let delta: u32 = self.deltas[block as usize].into(); - if delta > 0 { - Ok(self.digested_era.wrapping_add(delta)) + fn get(&self, block: u32) -> Option { + if let Some(&delta) = self.deltas.get(block as usize) { + let d: u32 = delta.into(); + if d == 0 { + None + } else { + Some(self.digested_era.wrapping_add(d)) + } } else { - Ok(0) + None } } } @@ -136,18 +141,14 @@ impl<'a> ArrayVisitor for LogicalEraEmitter<'a> { let begin = index as u32 * b.header.max_entries; let end = begin + b.header.nr_entries; for (v, block) in b.values.iter().zip(begin..end) { - let archived = inner - .era_archive - .get(block) - .map_err(|e| array::value_err(format!("{}", e)))?; - - let era = if archived > 0 { - ir::Era { + let era; + if let Some(archived) = inner.era_archive.get(block) { + era = ir::Era { block, era: archived, } } else { - ir::Era { block, era: *v } + era = ir::Era { block, era: *v } }; inner @@ -297,6 +298,41 @@ pub fn dump_metadata( //----------------------------------------- +fn get_writesets_ordered( + engine: Arc, + sb: &Superblock, + repair: bool, +) -> Result> { + let mut path = vec![0]; + let mut writesets = + btree_to_map::(&mut path, engine.clone(), repair, sb.writeset_tree_root)?; + + if sb.current_writeset.root != 0 { + if writesets.contains_key(&(sb.current_era as u64)) { + return Err(anyhow!( + "Duplicated era found in current_writeset and the writeset tree" + )); + } + writesets.insert(sb.current_era as u64, sb.current_writeset); + } + + if writesets.is_empty() { + return Ok(Vec::new()); + } + + let mut v = Vec::<(u32, Writeset)>::new(); + let era_begin = sb.current_era.wrapping_sub((writesets.len() - 1) as u32); + for era in era_begin..=sb.current_era { + if let Some(ws) = writesets.get(&(era as u64)) { + v.push((era, *ws)); + } else { + return Err(anyhow!("Writeset of era {} is not present", era)); + } + } + + Ok(v) +} + fn collate_writeset(index: u32, bitset: &FixedBitSet, archive: &mut dyn Archive) -> Result<()> { let era_delta = index + 1; @@ -320,12 +356,10 @@ fn collate_writesets( sb: &Superblock, repair: bool, ) -> Result> { - let mut path = vec![0]; - let writesets = - btree_to_map::(&mut path, engine.clone(), repair, sb.writeset_tree_root)?; + let writesets = get_writesets_ordered(engine.clone(), sb, repair)?; - let archived_min = writesets.iter().next().map_or(0u32, |(k, _v)| *k as u32); - let mut archive = new_era_archive(sb.nr_blocks, archived_min, writesets.len() as u32); + let archived_begin = writesets.get(0).map_or(0u32, |(era, _ws)| *era); + let mut archive = new_era_archive(sb.nr_blocks, archived_begin, writesets.len() as u32); for (index, (_era, ws)) in writesets.iter().enumerate() { let bitset = read_bitset_no_err(engine.clone(), ws.root, ws.nr_bits as usize, repair)?; @@ -377,14 +411,7 @@ pub fn dump(opts: EraDumpOptions) -> anyhow::Result<()> { } let mut out = xml::XmlWriter::new(writer, false); - let mut path = vec![0]; - let writesets = btree_to_key_set::( - &mut path, - ctx.engine.clone(), - opts.repair, - sb.writeset_tree_root, - )?; - + let writesets = get_writesets_ordered(ctx.engine.clone(), &sb, opts.repair)?; if opts.logical && !writesets.is_empty() { dump_metadata_logical(ctx.engine, &mut out, &sb, opts.repair) } else { From cc89cef43f5af5d4a8d6b62cfbf35b90f15bf1fb Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 8 Oct 2021 12:34:53 +0800 Subject: [PATCH 21/25] [era_dump (rust)] Dump the current_writeset that has not been archived This patch is added in corresponding to the dm-era patch de89afc1 in kernel: dm era: Recover committed writeset after crash --- src/era/dump.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/era/dump.rs b/src/era/dump.rs index d81ffd2..07c8abf 100644 --- a/src/era/dump.rs +++ b/src/era/dump.rs @@ -277,9 +277,7 @@ pub fn dump_metadata( }; out.superblock_b(&xml_sb)?; - let mut path = vec![0]; - let writesets = - btree_to_map::(&mut path, engine.clone(), repair, sb.writeset_tree_root)?; + let writesets = get_writesets_ordered(engine.clone(), sb, repair)?; for (era, ws) in writesets.iter() { dump_writeset(engine.clone(), out, *era as u32, ws, repair)?; } From 34f6b6fc62ac3bf2a260d923eeac83c9b83c4a7d Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 8 Oct 2021 22:02:57 +0800 Subject: [PATCH 22/25] [cache_metadata_size (rust)] First code drop --- src/bin/pdata_tools.rs | 2 + src/cache/metadata_size.rs | 19 +++++++ src/cache/mod.rs | 1 + src/commands/cache_metadata_size.rs | 88 +++++++++++++++++++++++++++++ src/commands/mod.rs | 1 + 5 files changed, 111 insertions(+) create mode 100644 src/cache/metadata_size.rs create mode 100644 src/commands/cache_metadata_size.rs diff --git a/src/bin/pdata_tools.rs b/src/bin/pdata_tools.rs index c2ababc..b3e0676 100644 --- a/src/bin/pdata_tools.rs +++ b/src/bin/pdata_tools.rs @@ -30,6 +30,8 @@ fn main_() -> Result<()> { cache_check::run(&new_args); } else if name_eq(name, "cache_dump") { cache_dump::run(&new_args); + } else if name_eq(name, "cache_metadata_size") { + cache_metadata_size::run(&new_args); } else if name_eq(name, "cache_repair") { cache_repair::run(&new_args); } else if name_eq(name, "cache_restore") { diff --git a/src/cache/metadata_size.rs b/src/cache/metadata_size.rs new file mode 100644 index 0000000..bedea4f --- /dev/null +++ b/src/cache/metadata_size.rs @@ -0,0 +1,19 @@ +use anyhow::Result; + +pub struct CacheMetadataSizeOptions { + pub nr_blocks: u64, + pub max_hint_width: u32, // bytes +} + +pub fn metadata_size(opts: &CacheMetadataSizeOptions) -> Result { + const SECTOR_SHIFT: u64 = 9; // 512 bytes per sector + const BYTES_PER_BLOCK_SHIFT: u64 = 4; // 16 bytes for key and value + const TRANSACTION_OVERHEAD: u64 = 8192; // in sectors; 4 MB + const HINT_OVERHEAD_PER_BLOCK: u64 = 8; // 8 bytes for the key + + let mapping_size = (opts.nr_blocks << BYTES_PER_BLOCK_SHIFT) >> SECTOR_SHIFT; + let hint_size = + (opts.nr_blocks * (opts.max_hint_width as u64 + HINT_OVERHEAD_PER_BLOCK)) >> SECTOR_SHIFT; + + Ok(TRANSACTION_OVERHEAD + mapping_size + hint_size) +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 4a90694..7f9075e 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -3,6 +3,7 @@ pub mod dump; pub mod hint; pub mod ir; pub mod mapping; +pub mod metadata_size; pub mod repair; pub mod restore; pub mod superblock; diff --git a/src/commands/cache_metadata_size.rs b/src/commands/cache_metadata_size.rs new file mode 100644 index 0000000..047d08c --- /dev/null +++ b/src/commands/cache_metadata_size.rs @@ -0,0 +1,88 @@ +extern crate clap; + +use clap::{value_t_or_exit, App, Arg, ArgGroup}; +use std::ffi::OsString; +use std::process; + +use crate::cache::metadata_size::{metadata_size, CacheMetadataSizeOptions}; +use crate::math::div_up; + +//------------------------------------------ + +fn parse_args(args: I) -> CacheMetadataSizeOptions +where + I: IntoIterator, + T: Into + Clone, +{ + let parser = App::new("cache_metadata_size") + .version(crate::version::tools_version()) + .about("Estimate the size of the metadata device needed for a given configuration.") + .usage("cache_metadata_size [OPTIONS] <--device-size --block-size | --nr-blocks >") + // options + .arg( + Arg::with_name("BLOCK_SIZE") + .help("Specify the size of each cache block") + .long("block-size") + .requires("DEVICE_SIZE") + .value_name("SECTORS"), + ) + .arg( + Arg::with_name("DEVICE_SIZE") + .help("Specify total size of the fast device used in the cache") + .long("device-size") + .requires("BLOCK_SIZE") + .value_name("SECTORS"), + ) + .arg( + Arg::with_name("NR_BLOCKS") + .help("Specify the number of cache blocks") + .long("nr-blocks") + .value_name("NUM"), + ) + .arg( + Arg::with_name("MAX_HINT_WIDTH") + .help("Specity the per-block hint width") + .long("max-hint-width") + .value_name("BYTES") + .default_value("4"), + ) + .group( + ArgGroup::with_name("selection") + .args(&["DEVICE_SIZE", "NR_BLOCKS"]) + .required(true) + ); + + let matches = parser.get_matches_from(args); + + let nr_blocks = matches.value_of("NR_BLOCKS").map_or_else( + || { + let device_size = value_t_or_exit!(matches.value_of("DEVICE_SIZE"), u64); + let block_size = value_t_or_exit!(matches.value_of("BLOCK_SIZE"), u32); + div_up(device_size, block_size as u64) + }, + |_| value_t_or_exit!(matches.value_of("NR_BLOCKS"), u64), + ); + + let max_hint_width = value_t_or_exit!(matches.value_of("MAX_HINT_WIDTH"), u32); + + CacheMetadataSizeOptions { + nr_blocks, + max_hint_width, + } +} + +pub fn run(args: &[std::ffi::OsString]) { + let opts = parse_args(args); + + match metadata_size(&opts) { + Ok(size) => { + println!("{} sectors", size); + } + Err(reason) => { + eprintln!("{}", reason); + process::exit(1); + } + } +} + +//------------------------------------------ diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 5dd1f19..34377a6 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,5 +1,6 @@ pub mod cache_check; pub mod cache_dump; +pub mod cache_metadata_size; pub mod cache_repair; pub mod cache_restore; pub mod era_check; From 9ea75ba113e3fea8c3c15af5da910079fd87dedb Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 12 Oct 2021 14:36:16 +0800 Subject: [PATCH 23/25] [tests] Move cache_metadata_size tests to Rust Changes for the Rust version in corresponding to command line changes: - Disable tests 'all_args_agree' and 'conradictory_info_fails' - Test conflicts between --nr-blocks and {--device-size|--block-size} --- functional-tests/cache-functional-tests.scm | 72 -------- tests/cache_metadata_size.rs | 186 ++++++++++++++++++++ tests/common/process.rs | 6 +- tests/common/target.rs | 8 + 4 files changed, 199 insertions(+), 73 deletions(-) create mode 100644 tests/cache_metadata_size.rs diff --git a/functional-tests/cache-functional-tests.scm b/functional-tests/cache-functional-tests.scm index f99606f..48fe646 100644 --- a/functional-tests/cache-functional-tests.scm +++ b/functional-tests/cache-functional-tests.scm @@ -58,76 +58,4 @@ (with-temp-file-sized ((md "cache.bin" 512)) (run-fail (cache-dump md)))) - - ;;;----------------------------------------------------------- - ;;; cache_metadata_size scenarios - ;;;----------------------------------------------------------- - - (define-scenario (cache-metadata-size v) - "cache_metadata_size -V" - (run-ok-rcv (stdout _) (cache-metadata-size "-V") - (assert-equal tools-version stdout))) - - (define-scenario (cache-metadata-size version) - "cache_metadata_size --version" - (run-ok-rcv (stdout _) (cache-metadata-size "--version") - (assert-equal tools-version stdout))) - - (define-scenario (cache-metadata-size h) - "cache_metadata_size -h" - (run-ok-rcv (stdout _) (cache-metadata-size "-h") - (assert-equal cache-metadata-size-help stdout))) - - (define-scenario (cache-metadata-size help) - "cache_metadata_size --help" - (run-ok-rcv (stdout _) (cache-metadata-size "--help") - (assert-equal cache-metadata-size-help stdout))) - - (define-scenario (cache-metadata-size no-args) - "No arguments specified causes fail" - (run-fail-rcv (_ stderr) (cache-metadata-size) - (assert-equal "Please specify either --device-size and --block-size, or --nr-blocks." - stderr))) - - (define-scenario (cache-metadata-size device-size-only) - "Just --device-size causes fail" - (run-fail-rcv (_ stderr) (cache-metadata-size "--device-size" (to-bytes (meg 100))) - (assert-equal "If you specify --device-size you must also give --block-size." - stderr))) - - (define-scenario (cache-metadata-size block-size-only) - "Just --block-size causes fail" - (run-fail-rcv (_ stderr) (cache-metadata-size "--block-size" 128) - (assert-equal "If you specify --block-size you must also give --device-size." - stderr))) - - (define-scenario (cache-metadata-size conradictory-info-fails) - "Contradictory info causes fail" - (run-fail-rcv (_ stderr) (cache-metadata-size "--device-size 102400 --block-size 1000 --nr-blocks 6") - (assert-equal "Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size." - stderr))) - - (define-scenario (cache-metadata-size all-args-agree) - "All args agreeing succeeds" - (run-ok-rcv (stdout stderr) (cache-metadata-size "--device-size" 102400 "--block-size" 100 "--nr-blocks" 1024) - (assert-equal "8248 sectors" stdout) - (assert-eof stderr))) - - (define-scenario (cache-metadata-size nr-blocks-alone) - "Just --nr-blocks succeeds" - (run-ok-rcv (stdout stderr) (cache-metadata-size "--nr-blocks" 1024) - (assert-equal "8248 sectors" stdout) - (assert-eof stderr))) - - (define-scenario (cache-metadata-size dev-size-and-block-size-succeeds) - "Specifying --device-size with --block-size succeeds" - (run-ok-rcv (stdout stderr) (cache-metadata-size "--device-size" 102400 "--block-size" 100) - (assert-equal "8248 sectors" stdout) - (assert-eof stderr))) - - (define-scenario (cache-metadata-size big-config) - "A big configuration succeeds" - (run-ok-rcv (stdout stderr) (cache-metadata-size "--nr-blocks 67108864") - (assert-equal "3678208 sectors" stdout) - (assert-eof stderr))) ) diff --git a/tests/cache_metadata_size.rs b/tests/cache_metadata_size.rs new file mode 100644 index 0000000..22ea6cf --- /dev/null +++ b/tests/cache_metadata_size.rs @@ -0,0 +1,186 @@ +use anyhow::Result; + +mod common; + +use common::common_args::*; +use common::process::*; +use common::program::*; +use common::target::*; + +//------------------------------------------ + +const USAGE: &str = "cache_metadata_size 0.9.0 +Estimate the size of the metadata device needed for a given configuration. + +USAGE: + cache_metadata_size [OPTIONS] <--device-size --block-size | --nr-blocks > + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + --block-size Specify the size of each cache block + --device-size Specify total size of the fast device used in the cache + --max-hint-width Specity the per-block hint width [default: 4] + --nr-blocks Specify the number of cache blocks"; + +//------------------------------------------ + +struct CacheMetadataSize; + +impl<'a> Program<'a> for CacheMetadataSize { + fn name() -> &'a str { + "cache_metadata_size" + } + + fn cmd(args: I) -> Command + where + I: IntoIterator, + I::Item: Into, + { + cache_metadata_size_cmd(args) + } + + fn usage() -> &'a str { + USAGE + } + + fn arg_type() -> ArgType { + ArgType::InputArg + } + + fn bad_option_hint(option: &str) -> String { + msg::bad_option_hint(option) + } +} + +//------------------------------------------ + +test_accepts_help!(CacheMetadataSize); +test_accepts_version!(CacheMetadataSize); +test_rejects_bad_option!(CacheMetadataSize); + +//------------------------------------------ + +#[test] +fn no_args() -> Result<()> { + let _stderr = run_fail(cache_metadata_size_cmd([""; 0]))?; + Ok(()) +} + +#[test] +fn device_size_only() -> Result<()> { + let _stderr = run_fail(cache_metadata_size_cmd(args!["--device-size", "204800"]))?; + Ok(()) +} + +#[test] +fn block_size_only() -> Result<()> { + let _stderr = run_fail(cache_metadata_size_cmd(args!["--block-size", "128"]))?; + Ok(()) +} + +/* +#[test] +fn conradictory_info_fails() -> Result<()> { + let stderr = run_fail(cache_metadata_size_cmd( + args![ + "--device-size", + "102400", + "--block-size", + "1000", + "--nr-blocks", + "6" + ], + ))?; + assert_eq!(stderr, "Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size."); + Ok(()) +} + +#[test] +fn all_args_agree() -> Result<()> { + let out = run_ok_raw(cache_metadata_size_cmd( + args![ + "--device-size", + "102400", + "--block-size", + "100", + "--nr-blocks", + "1024" + ], + ))?; + let stdout = std::str::from_utf8(&out.stdout[..]) + .unwrap() + .trim_end_matches(|c| c == '\n' || c == '\r') + .to_string(); + assert_eq!(stdout, "8248 sectors"); + assert_eq!(out.stderr.len(), 0); + Ok(()) +} +*/ + +#[test] +fn dev_size_and_nr_blocks_conflicts() -> Result<()> { + run_fail(cache_metadata_size_cmd(args![ + "--device-size", + "102400", + "--nr-blocks", + "1024" + ]))?; + Ok(()) +} + +#[test] +fn block_size_and_nr_blocks_conflicts() -> Result<()> { + run_fail(cache_metadata_size_cmd(args![ + "--block-size", + "100", + "--nr-blocks", + "1024" + ]))?; + Ok(()) +} + +#[test] +fn nr_blocks_alone() -> Result<()> { + let out = run_ok_raw(cache_metadata_size_cmd(args!["--nr-blocks", "1024"]))?; + let stdout = std::str::from_utf8(&out.stdout[..]) + .unwrap() + .trim_end_matches(|c| c == '\n' || c == '\r') + .to_string(); + assert_eq!(stdout, "8248 sectors"); + assert_eq!(out.stderr.len(), 0); + Ok(()) +} + +#[test] +fn dev_size_and_block_size_succeeds() -> Result<()> { + let out = run_ok_raw(cache_metadata_size_cmd(args![ + "--device-size", + "102400", + "--block-size", + "100" + ]))?; + let stdout = std::str::from_utf8(&out.stdout[..]) + .unwrap() + .trim_end_matches(|c| c == '\n' || c == '\r') + .to_string(); + assert_eq!(stdout, "8248 sectors"); + assert_eq!(out.stderr.len(), 0); + Ok(()) +} + +#[test] +fn large_nr_blocks() -> Result<()> { + let out = run_ok_raw(cache_metadata_size_cmd(args!["--nr-blocks", "67108864"]))?; + let stdout = std::str::from_utf8(&out.stdout[..]) + .unwrap() + .trim_end_matches(|c| c == '\n' || c == '\r') + .to_string(); + assert_eq!(stdout, "3678208 sectors"); + assert_eq!(out.stderr.len(), 0); + Ok(()) +} + +//------------------------------------------ diff --git a/tests/common/process.rs b/tests/common/process.rs index 790b9ef..1cbea92 100644 --- a/tests/common/process.rs +++ b/tests/common/process.rs @@ -100,7 +100,11 @@ pub fn run_fail(command: Command) -> Result { let output = command.unchecked().run()?; log_output(&output); assert!(!output.status.success()); - let stderr = std::str::from_utf8(&output.stderr[..]).unwrap().to_string(); + let stderr = std::str::from_utf8(&output.stderr[..]) + .unwrap() + .trim_end_matches(|c| c == '\n' || c == '\r') + .to_string(); + Ok(stderr) } diff --git a/tests/common/target.rs b/tests/common/target.rs index fa8ffc0..f688c22 100644 --- a/tests/common/target.rs +++ b/tests/common/target.rs @@ -143,6 +143,14 @@ where rust_cmd("cache_dump", args) } +pub fn cache_metadata_size_cmd(args: I) -> Command +where + I: IntoIterator, + I::Item: Into, +{ + rust_cmd("cache_metadata_size", args) +} + pub fn cache_restore_cmd(args: I) -> Command where I: IntoIterator, From e5f0acd288900033ef8d06dd06ac3c61e3bf7f6e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 14 Oct 2021 22:14:33 +0800 Subject: [PATCH 24/25] [thin_metadata_size (rust)] First code drop --- src/bin/pdata_tools.rs | 2 + src/commands/mod.rs | 1 + src/commands/thin_metadata_size.rs | 100 +++++++++++++++++++++++++++ src/lib.rs | 1 + src/thin/metadata_size.rs | 21 ++++++ src/thin/mod.rs | 1 + src/units.rs | 104 +++++++++++++++++++++++++++++ 7 files changed, 230 insertions(+) create mode 100644 src/commands/thin_metadata_size.rs create mode 100644 src/thin/metadata_size.rs create mode 100644 src/units.rs diff --git a/src/bin/pdata_tools.rs b/src/bin/pdata_tools.rs index b3e0676..da5d430 100644 --- a/src/bin/pdata_tools.rs +++ b/src/bin/pdata_tools.rs @@ -48,6 +48,8 @@ fn main_() -> Result<()> { thin_dump::run(&new_args); } else if name_eq(name, "thin_metadata_pack") { thin_metadata_pack::run(&new_args); + } else if name_eq(name, "thin_metadata_size") { + thin_metadata_size::run(&new_args); } else if name_eq(name, "thin_metadata_unpack") { thin_metadata_unpack::run(&new_args); } else if name_eq(name, "thin_repair") { diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 34377a6..74fc092 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -11,6 +11,7 @@ pub mod era_restore; pub mod thin_check; pub mod thin_dump; pub mod thin_metadata_pack; +pub mod thin_metadata_size; pub mod thin_metadata_unpack; pub mod thin_repair; pub mod thin_restore; diff --git a/src/commands/thin_metadata_size.rs b/src/commands/thin_metadata_size.rs new file mode 100644 index 0000000..1574d5a --- /dev/null +++ b/src/commands/thin_metadata_size.rs @@ -0,0 +1,100 @@ +extern crate clap; + +use clap::{value_t_or_exit, App, Arg}; +use std::ffi::OsString; +use std::process; + +use crate::thin::metadata_size::{metadata_size, ThinMetadataSizeOptions}; +use crate::units::*; + +//------------------------------------------ + +fn parse_args(args: I) -> (ThinMetadataSizeOptions, Units, bool) +where + I: IntoIterator, + T: Into + Clone, +{ + let parser = App::new("thin_metadata_size") + .version(crate::version::tools_version()) + .about("Estimate the size of the metadata device needed for a given configuration.") + // options + .arg( + Arg::with_name("BLOCK_SIZE") + .help("Specify the data block size") + .short("b") + .long("block-size") + .required(true) + .value_name("SECTORS"), + ) + .arg( + Arg::with_name("POOL_SIZE") + .help("Specify the size of pool device") + .short("s") + .long("pool-size") + .required(true) + .value_name("SECTORS"), + ) + .arg( + Arg::with_name("MAX_THINS") + .help("Maximum number of thin devices and snapshots") + .short("m") + .long("max-thins") + .required(true) + .value_name("NUM"), + ) + .arg( + Arg::with_name("UNIT") + .help("Specify the output unit") + .short("u") + .long("unit") + .value_name("UNIT") + .default_value("sector"), + ) + .arg( + Arg::with_name("NUMERIC_ONLY") + .help("Output numeric value only") + .short("n") + .long("numeric-only"), + ); + + let matches = parser.get_matches_from(args); + + // TODO: handle unit suffix + let pool_size = value_t_or_exit!(matches.value_of("POOL_SIZE"), u64); + let block_size = value_t_or_exit!(matches.value_of("BLOCK_SIZE"), u32); + let max_thins = value_t_or_exit!(matches.value_of("MAX_THINS"), u64); + let unit = value_t_or_exit!(matches.value_of("UNIT"), Units); + let numeric_only = matches.is_present("NUMERIC_ONLY"); + + ( + ThinMetadataSizeOptions { + nr_blocks: pool_size / block_size as u64, + max_thins, + }, + unit, + numeric_only, + ) +} + +pub fn run(args: &[std::ffi::OsString]) { + let (opts, unit, numeric_only) = parse_args(args); + + match metadata_size(&opts) { + Ok(size) => { + let size = to_units(size * 512, unit.clone()); + if numeric_only { + println!("{}", size); + } else { + let mut name = unit.to_string(); + name.push('s'); + println!("{} {}", size, name); + } + } + Err(reason) => { + eprintln!("{}", reason); + process::exit(1); + } + } +} + +//------------------------------------------ diff --git a/src/lib.rs b/src/lib.rs index 23fc9c3..ef9cef1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,6 +27,7 @@ pub mod pdata; pub mod report; pub mod shrink; pub mod thin; +pub mod units; pub mod version; pub mod write_batcher; pub mod xml; diff --git a/src/thin/metadata_size.rs b/src/thin/metadata_size.rs new file mode 100644 index 0000000..340ffd1 --- /dev/null +++ b/src/thin/metadata_size.rs @@ -0,0 +1,21 @@ +use anyhow::Result; + +use crate::math::div_up; + +pub struct ThinMetadataSizeOptions { + pub nr_blocks: u64, + pub max_thins: u64, +} + +pub fn metadata_size(opts: &ThinMetadataSizeOptions) -> Result { + const ENTRIES_PER_NODE: u64 = 126; // assumed the mapping leaves are half populated + const BLOCK_SIZE: u64 = 8; // sectors + + // size of all the leaf nodes for data mappings + let mapping_size = div_up(opts.nr_blocks, ENTRIES_PER_NODE) * BLOCK_SIZE; + + // space required by root nodes + let roots_overhead = opts.max_thins * BLOCK_SIZE; + + Ok(mapping_size + roots_overhead) +} diff --git a/src/thin/mod.rs b/src/thin/mod.rs index 197f2c2..71928e7 100644 --- a/src/thin/mod.rs +++ b/src/thin/mod.rs @@ -5,6 +5,7 @@ pub mod dump; pub mod ir; pub mod metadata; pub mod metadata_repair; +pub mod metadata_size; pub mod repair; pub mod restore; pub mod runs; diff --git a/src/units.rs b/src/units.rs new file mode 100644 index 0000000..11a4527 --- /dev/null +++ b/src/units.rs @@ -0,0 +1,104 @@ +use anyhow::anyhow; +use std::str::FromStr; + +//------------------------------------------ + +#[derive(Clone)] +pub enum Units { + Byte, + Sector, + Kilobyte, + Megabyte, + Gigabyte, + Terabyte, + Petabyte, + Exabyte, + Kibibyte, + Mebibyte, + Gibibyte, + Tebibyte, + Pebibyte, + Exbibyte, +} + +impl Units { + fn size_bytes(&self) -> u64 { + match self { + Units::Byte => 1, + Units::Sector => 512, + // base 2 + Units::Kibibyte => 1024, + Units::Mebibyte => 1048576, + Units::Gibibyte => 1073741824, + Units::Tebibyte => 1099511627776, + Units::Pebibyte => 1125899906842624, + Units::Exbibyte => 1152921504606846976, + // base 10 + Units::Kilobyte => 1000, + Units::Megabyte => 1000000, + Units::Gigabyte => 1000000000, + Units::Terabyte => 1000000000000, + Units::Petabyte => 1000000000000000, + Units::Exabyte => 1000000000000000000, + } + } +} + +impl FromStr for Units { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "byte" | "b" => Ok(Units::Byte), + "sector" | "s" => Ok(Units::Sector), + // base 2 + "kibibyte" | "k" => Ok(Units::Kibibyte), + "mibibyte" | "m" => Ok(Units::Mebibyte), + "gibibyte" | "g" => Ok(Units::Gibibyte), + "tebibyte" | "t" => Ok(Units::Tebibyte), + "pebibyte" | "p" => Ok(Units::Pebibyte), + "exbibyte" | "e" => Ok(Units::Exbibyte), + // base 10 + "kilobyte" | "K" => Ok(Units::Kilobyte), + "megabyte" | "M" => Ok(Units::Megabyte), + "gigabyte" | "G" => Ok(Units::Gigabyte), + "terabyte" | "T" => Ok(Units::Terabyte), + "petabyte" | "P" => Ok(Units::Petabyte), + "exabyte" | "E" => Ok(Units::Exabyte), + _ => Err(anyhow!("Invalid unit specifier")), + } + } +} + +impl ToString for Units { + fn to_string(&self) -> String { + String::from(match self { + Units::Byte => "byte", + Units::Sector => "sector", + // base 2 + Units::Kibibyte => "kibibyte", + Units::Mebibyte => "mibibyte", + Units::Gibibyte => "gibibyte", + Units::Tebibyte => "terabyte", + Units::Pebibyte => "pebibyte", + Units::Exbibyte => "exbibyte", + // base 10 + Units::Kilobyte => "kilobyte", + Units::Megabyte => "megabyte", + Units::Gigabyte => "gigabyte", + Units::Terabyte => "terabyte", + Units::Petabyte => "petabyte", + Units::Exabyte => "exabyte", + }) + } +} + +pub fn to_bytes(size: u64, unit: Units) -> u64 { + size * unit.size_bytes() +} + +pub fn to_units(bytes: u64, unit: Units) -> f64 { + bytes as f64 / unit.size_bytes() as f64 +} + +//------------------------------------------ From f3c2ade90acbd1ac9a9391ff8e09fbe252a63b45 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 12 Oct 2021 23:19:00 +0800 Subject: [PATCH 25/25] [thin_ll_dump] Fix potential segfault while reading invalid subtree roots --- thin-provisioning/thin_ll_dump.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/thin-provisioning/thin_ll_dump.cc b/thin-provisioning/thin_ll_dump.cc index e6048d0..9207cd2 100644 --- a/thin-provisioning/thin_ll_dump.cc +++ b/thin-provisioning/thin_ll_dump.cc @@ -135,7 +135,9 @@ namespace { public: ll_mapping_tree_emitter(block_manager::ptr bm, indented_stream &out) - : bm_(bm), out_(out) { + : bm_(bm), + nv_(create_btree_node_validator()), + out_(out) { } void visit(btree_path const &path, block_address tree_root) { @@ -147,6 +149,7 @@ namespace { try { block_manager::read_ref rr = bm_->read_lock(tree_root); node_ref n = btree_detail::to_node(rr); + nv_->check(n.raw(), tree_root); node_info ni; convert_to_node_info(n, ni); output_node_info(out_, ni); @@ -160,6 +163,7 @@ namespace { } private: block_manager::ptr bm_; + bcache::validator::ptr nv_; indented_stream& out_; };