From 7002a8ae8d66c83f2db600a6f987ed92a9e3002b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 8 Jul 2021 17:04:40 +0800 Subject: [PATCH 1/8] [all (rust)] Pull out structures for intermediate representation Also fix the data type for thin timestamp --- src/cache/dump.rs | 11 ++--- src/cache/ir.rs | 60 ++++++++++++++++++++++++++ src/cache/mod.rs | 1 + src/cache/restore.rs | 15 ++++--- src/cache/xml.rs | 54 +---------------------- src/shrink/toplevel.rs | 23 +++++----- src/thin/dump.rs | 31 +++++++------- src/thin/ir.rs | 60 ++++++++++++++++++++++++++ src/thin/mod.rs | 1 + src/thin/restore.rs | 13 +++--- src/thin/xml.rs | 66 +++-------------------------- tests/common/cache_xml_generator.rs | 9 ++-- tests/common/thin_xml_generator.rs | 29 +++++++------ tests/thin_shrink.rs | 11 ++--- 14 files changed, 205 insertions(+), 179 deletions(-) create mode 100644 src/cache/ir.rs create mode 100644 src/thin/ir.rs diff --git a/src/cache/dump.rs b/src/cache/dump.rs index a3685f5..c5c33c1 100644 --- a/src/cache/dump.rs +++ b/src/cache/dump.rs @@ -7,9 +7,10 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use crate::cache::hint::Hint; +use crate::cache::ir::{self, MetadataVisitor}; use crate::cache::mapping::Mapping; use crate::cache::superblock::*; -use crate::cache::xml::{self, MetadataVisitor}; +use crate::cache::xml; use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::*; @@ -58,7 +59,7 @@ mod format1 { continue; } - let m = xml::Map { + let m = ir::Map { cblock, oblock: map.oblock, dirty: map.is_dirty(), @@ -133,7 +134,7 @@ mod format2 { // default to dirty if the bitset is damaged dirty = true; } - let m = xml::Map { + let m = ir::Map { cblock, oblock: map.oblock, dirty, @@ -175,7 +176,7 @@ impl<'a> ArrayVisitor for HintEmitter<'a> { continue; } - let h = xml::Hint { + let h = ir::Hint { cblock, data: hint.hint.to_vec(), }; @@ -226,7 +227,7 @@ fn dump_metadata( let engine = &ctx.engine; let mut out = xml::XmlWriter::new(w); - let xml_sb = xml::Superblock { + let xml_sb = ir::Superblock { uuid: "".to_string(), block_size: sb.data_block_size, nr_cache_blocks: sb.cache_blocks, diff --git a/src/cache/ir.rs b/src/cache/ir.rs new file mode 100644 index 0000000..65be4aa --- /dev/null +++ b/src/cache/ir.rs @@ -0,0 +1,60 @@ +use anyhow::Result; + +//------------------------------------------ + +#[derive(Clone)] +pub struct Superblock { + pub uuid: String, + pub block_size: u32, + pub nr_cache_blocks: u32, + pub policy: String, + pub hint_width: u32, +} + +#[derive(Clone)] +pub struct Map { + pub cblock: u32, + pub oblock: u64, + pub dirty: bool, +} + +#[derive(Clone)] +pub struct Hint { + pub cblock: u32, + pub data: Vec, +} + +#[derive(Clone)] +pub struct Discard { + pub begin: u64, + pub end: u64, +} + +//------------------------------------------ + +#[derive(Clone)] +pub enum Visit { + Continue, + Stop, +} + +pub trait MetadataVisitor { + fn superblock_b(&mut self, sb: &Superblock) -> Result; + fn superblock_e(&mut self) -> Result; + + fn mappings_b(&mut self) -> Result; + fn mappings_e(&mut self) -> Result; + fn mapping(&mut self, m: &Map) -> Result; + + fn hints_b(&mut self) -> Result; + fn hints_e(&mut self) -> Result; + fn hint(&mut self, h: &Hint) -> Result; + + fn discards_b(&mut self) -> Result; + fn discards_e(&mut self) -> Result; + fn discard(&mut self, d: &Discard) -> Result; + + fn eof(&mut self) -> Result; +} + +//------------------------------------------ diff --git a/src/cache/mod.rs b/src/cache/mod.rs index ceb8867..c01cc6c 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1,6 +1,7 @@ pub mod check; pub mod dump; pub mod hint; +pub mod ir; pub mod mapping; pub mod restore; pub mod superblock; diff --git a/src/cache/restore.rs b/src/cache/restore.rs index e7ab5cd..5d5f23c 100644 --- a/src/cache/restore.rs +++ b/src/cache/restore.rs @@ -7,9 +7,10 @@ use std::path::Path; use std::sync::Arc; use crate::cache::hint::Hint; +use crate::cache::ir::{self, MetadataVisitor, Visit}; use crate::cache::mapping::{Mapping, MappingFlags}; use crate::cache::superblock::*; -use crate::cache::xml::{self, MetadataVisitor, Visit}; +use crate::cache::xml; use crate::io_engine::*; use crate::math::*; use crate::pdata::array_builder::*; @@ -56,7 +57,7 @@ fn mk_context(opts: &CacheRestoreOptions) -> anyhow::Result { //------------------------------------------ struct RestoreResult { - sb: xml::Superblock, + sb: ir::Superblock, mapping_root: u64, dirty_root: Option, hint_root: u64, @@ -65,7 +66,7 @@ struct RestoreResult { struct Restorer<'a> { write_batcher: &'a mut WriteBatcher, - sb: Option, + sb: Option, mapping_builder: Option>, dirty_builder: Option>, hint_builder: Option>, @@ -112,7 +113,7 @@ impl<'a> Restorer<'a> { } impl<'a> MetadataVisitor for Restorer<'a> { - fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { + fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { self.sb = Some(sb.clone()); self.write_batcher.alloc()?; self.mapping_builder = Some(ArrayBuilder::new(sb.nr_cache_blocks as u64)); @@ -157,7 +158,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { Ok(Visit::Continue) } - fn mapping(&mut self, m: &xml::Map) -> Result { + fn mapping(&mut self, m: &ir::Map) -> Result { let map = Mapping { oblock: m.oblock, flags: MappingFlags::Valid as u32, @@ -198,7 +199,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { Ok(Visit::Continue) } - fn hint(&mut self, h: &xml::Hint) -> Result { + fn hint(&mut self, h: &ir::Hint) -> Result { let hint = Hint { hint: h.data[..].try_into().unwrap(), }; @@ -215,7 +216,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { Ok(Visit::Continue) } - fn discard(&mut self, _d: &xml::Discard) -> Result { + fn discard(&mut self, _d: &ir::Discard) -> Result { Ok(Visit::Continue) } diff --git a/src/cache/xml.rs b/src/cache/xml.rs index 6a05744..f171151 100644 --- a/src/cache/xml.rs +++ b/src/cache/xml.rs @@ -6,63 +6,11 @@ use std::io::{Read, Write}; use quick_xml::events::{BytesEnd, BytesStart, Event}; use quick_xml::{Reader, Writer}; +use crate::cache::ir::*; use crate::xml::*; //--------------------------------------- -#[derive(Clone)] -pub struct Superblock { - pub uuid: String, - pub block_size: u32, - pub nr_cache_blocks: u32, - pub policy: String, - pub hint_width: u32, -} - -#[derive(Clone)] -pub struct Map { - pub cblock: u32, - pub oblock: u64, - pub dirty: bool, -} - -#[derive(Clone)] -pub struct Hint { - pub cblock: u32, - pub data: Vec, -} - -#[derive(Clone)] -pub struct Discard { - pub begin: u64, - pub end: u64, -} - -#[derive(Clone)] -pub enum Visit { - Continue, - Stop, -} - -pub trait MetadataVisitor { - fn superblock_b(&mut self, sb: &Superblock) -> Result; - fn superblock_e(&mut self) -> Result; - - fn mappings_b(&mut self) -> Result; - fn mappings_e(&mut self) -> Result; - fn mapping(&mut self, m: &Map) -> Result; - - fn hints_b(&mut self) -> Result; - fn hints_e(&mut self) -> Result; - fn hint(&mut self, h: &Hint) -> Result; - - fn discards_b(&mut self) -> Result; - fn discards_e(&mut self) -> Result; - fn discard(&mut self, d: &Discard) -> Result; - - fn eof(&mut self) -> Result; -} - pub struct XmlWriter { w: Writer, } diff --git a/src/shrink/toplevel.rs b/src/shrink/toplevel.rs index e7cde54..fb47773 100644 --- a/src/shrink/toplevel.rs +++ b/src/shrink/toplevel.rs @@ -6,7 +6,8 @@ use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use crate::shrink::copier::{self, Region}; -use crate::thin::xml::{self, Visit}; +use crate::thin::ir::{self, MetadataVisitor, Visit}; +use crate::thin::xml; //--------------------------------------- @@ -34,8 +35,8 @@ impl Pass1 { } } -impl xml::MetadataVisitor for Pass1 { - fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { +impl MetadataVisitor for Pass1 { + fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { self.allocated_blocks.grow(sb.nr_data_blocks as usize); self.block_size = Some(sb.data_block_size as u64); Ok(Visit::Continue) @@ -53,7 +54,7 @@ impl xml::MetadataVisitor for Pass1 { todo!(); } - fn device_b(&mut self, _d: &xml::Device) -> Result { + fn device_b(&mut self, _d: &ir::Device) -> Result { Ok(Visit::Continue) } @@ -61,7 +62,7 @@ impl xml::MetadataVisitor for Pass1 { Ok(Visit::Continue) } - fn map(&mut self, m: &xml::Map) -> Result { + fn map(&mut self, m: &ir::Map) -> Result { for i in m.data_begin..(m.data_begin + m.len) { if i > self.nr_blocks { self.nr_high_blocks += 1; @@ -99,8 +100,8 @@ impl Pass2 { } } -impl xml::MetadataVisitor for Pass2 { - fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { +impl MetadataVisitor for Pass2 { + fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { self.writer.superblock_b(sb) } @@ -116,7 +117,7 @@ impl xml::MetadataVisitor for Pass2 { todo!(); } - fn device_b(&mut self, d: &xml::Device) -> Result { + fn device_b(&mut self, d: &ir::Device) -> Result { self.writer.device_b(d) } @@ -124,7 +125,7 @@ impl xml::MetadataVisitor for Pass2 { self.writer.device_e() } - fn map(&mut self, m: &xml::Map) -> Result { + fn map(&mut self, m: &ir::Map) -> Result { if m.data_begin + m.len < self.nr_blocks { // no remapping needed. self.writer.map(m)?; @@ -134,7 +135,7 @@ impl xml::MetadataVisitor for Pass2 { let mut written = 0; for r in remaps { - self.writer.map(&xml::Map { + self.writer.map(&ir::Map { thin_begin: m.thin_begin + written, data_begin: r.start, time: m.time, @@ -493,7 +494,7 @@ fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> V rs } -fn process_xml(input_path: &Path, pass: &mut MV) -> Result<()> { +fn process_xml(input_path: &Path, pass: &mut MV) -> Result<()> { let input = OpenOptions::new() .read(true) .write(false) diff --git a/src/thin/dump.rs b/src/thin/dump.rs index aa5942e..7296dfb 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -18,14 +18,15 @@ use crate::pdata::unpack::*; use crate::report::*; use crate::thin::block_time::*; use crate::thin::device_detail::*; +use crate::thin::ir::{self, MetadataVisitor}; use crate::thin::runs::*; use crate::thin::superblock::*; -use crate::thin::xml::{self, MetadataVisitor}; +use crate::thin::xml; //------------------------------------------ struct RunBuilder { - run: Option, + run: Option, } impl RunBuilder { @@ -33,12 +34,12 @@ impl RunBuilder { RunBuilder { run: None } } - fn next(&mut self, thin_block: u64, data_block: u64, time: u32) -> Option { - use xml::Map; + fn next(&mut self, thin_block: u64, data_block: u64, time: u32) -> Option { + use ir::Map; match self.run { None => { - self.run = Some(xml::Map { + self.run = Some(ir::Map { thin_begin: thin_block, data_begin: data_block, time, @@ -46,7 +47,7 @@ impl RunBuilder { }); None } - Some(xml::Map { + Some(ir::Map { thin_begin, data_begin, time: mtime, @@ -70,7 +71,7 @@ impl RunBuilder { } } - fn complete(&mut self) -> Option { + fn complete(&mut self) -> Option { self.run.take() } } @@ -78,7 +79,7 @@ impl RunBuilder { //------------------------------------------ struct MVInner<'a> { - md_out: &'a mut dyn xml::MetadataVisitor, + md_out: &'a mut dyn MetadataVisitor, builder: RunBuilder, } @@ -89,7 +90,7 @@ struct MappingVisitor<'a> { //------------------------------------------ impl<'a> MappingVisitor<'a> { - fn new(md_out: &'a mut dyn xml::MetadataVisitor) -> MappingVisitor<'a> { + fn new(md_out: &'a mut dyn MetadataVisitor) -> MappingVisitor<'a> { MappingVisitor { inner: Mutex::new(MVInner { md_out, @@ -516,7 +517,7 @@ where Ok(()) } -fn emit_leaves(ctx: &Context, out: &mut dyn xml::MetadataVisitor, ls: &[u64]) -> Result<()> { +fn emit_leaves(ctx: &Context, out: &mut dyn MetadataVisitor, ls: &[u64]) -> Result<()> { let mut v = MappingVisitor::new(out); let proc = |b| { emit_leaf(&mut v, &b)?; @@ -560,9 +561,9 @@ fn emit_entries( fn dump_metadata(ctx: &Context, w: &mut dyn Write, sb: &Superblock, md: &Metadata) -> Result<()> { let data_root = unpack::(&sb.data_sm_root[0..])?; let mut out = xml::XmlWriter::new(w); - let xml_sb = xml::Superblock { + let xml_sb = ir::Superblock { uuid: "".to_string(), - time: sb.time as u64, + time: sb.time, transaction: sb.transaction_id, flags: None, version: Some(2), @@ -581,12 +582,12 @@ fn dump_metadata(ctx: &Context, w: &mut dyn Write, sb: &Superblock, md: &Metadat ctx.report.set_title("Dumping devices"); for dev in &md.devs { - let device = xml::Device { + let device = ir::Device { dev_id: dev.thin_id, mapped_blocks: dev.detail.mapped_blocks, transaction: dev.detail.transaction_id, - creation_time: dev.detail.creation_time as u64, - snap_time: dev.detail.snapshotted_time as u64, + creation_time: dev.detail.creation_time, + snap_time: dev.detail.snapshotted_time, }; out.device_b(&device)?; emit_entries(ctx, &mut out, &dev.map.entries)?; diff --git a/src/thin/ir.rs b/src/thin/ir.rs new file mode 100644 index 0000000..ed4aa26 --- /dev/null +++ b/src/thin/ir.rs @@ -0,0 +1,60 @@ +use anyhow::Result; + +//------------------------------------------ + +#[derive(Clone)] +pub struct Superblock { + pub uuid: String, + pub time: u32, + pub transaction: u64, + pub flags: Option, + pub version: Option, + pub data_block_size: u32, + pub nr_data_blocks: u64, + pub metadata_snap: Option, +} + +#[derive(Clone)] +pub struct Device { + pub dev_id: u32, + pub mapped_blocks: u64, + pub transaction: u64, + pub creation_time: u32, + pub snap_time: u32, +} + +#[derive(Clone)] +pub struct Map { + pub thin_begin: u64, + pub data_begin: u64, + pub time: u32, + pub len: u64, +} + +//------------------------------------------ + +#[derive(Clone)] +pub enum Visit { + Continue, + Stop, +} + +pub trait MetadataVisitor { + fn superblock_b(&mut self, sb: &Superblock) -> Result; + fn superblock_e(&mut self) -> Result; + + // Defines a shared sub tree. May only contain a 'map' (no 'ref' allowed). + fn def_shared_b(&mut self, name: &str) -> Result; + fn def_shared_e(&mut self) -> Result; + + // A device contains a number of 'map' or 'ref' items. + fn device_b(&mut self, d: &Device) -> Result; + fn device_e(&mut self) -> Result; + + fn map(&mut self, m: &Map) -> Result; + fn ref_shared(&mut self, name: &str) -> Result; + + fn eof(&mut self) -> Result; +} + +//------------------------------------------ diff --git a/src/thin/mod.rs b/src/thin/mod.rs index db3dfc4..cbaa118 100644 --- a/src/thin/mod.rs +++ b/src/thin/mod.rs @@ -2,6 +2,7 @@ pub mod block_time; pub mod check; pub mod device_detail; pub mod dump; +pub mod ir; pub mod restore; pub mod runs; pub mod superblock; diff --git a/src/thin/restore.rs b/src/thin/restore.rs index 5b9dea4..cab683a 100644 --- a/src/thin/restore.rs +++ b/src/thin/restore.rs @@ -16,8 +16,9 @@ use crate::pdata::unpack::Pack; use crate::report::*; use crate::thin::block_time::*; use crate::thin::device_detail::*; +use crate::thin::ir::{self, MetadataVisitor, Visit}; use crate::thin::superblock::{self, *}; -use crate::thin::xml::{self, *}; +use crate::thin::xml; use crate::write_batcher::*; //------------------------------------------ @@ -58,7 +59,7 @@ impl std::fmt::Display for MappedSection { //------------------------------------------ struct RestoreResult { - sb: xml::Superblock, + sb: ir::Superblock, devices: BTreeMap, data_sm: Arc>, } @@ -74,7 +75,7 @@ struct Restorer<'a> { current_map: Option<(MappedSection, NodeBuilder)>, current_dev: Option, - sb: Option, + sb: Option, devices: BTreeMap, data_sm: Option>>, } @@ -136,7 +137,7 @@ impl<'a> Restorer<'a> { } impl<'a> MetadataVisitor for Restorer<'a> { - fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { + fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { self.sb = Some(sb.clone()); self.data_sm = Some(core_sm(sb.nr_data_blocks, u32::MAX)); self.w.alloc()?; @@ -160,7 +161,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { } } - fn device_b(&mut self, d: &Device) -> Result { + fn device_b(&mut self, d: &ir::Device) -> Result { self.report .info(&format!("building btree for device {}", d.dev_id)); self.current_dev = Some(DeviceDetail { @@ -186,7 +187,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { } } - fn map(&mut self, m: &Map) -> Result { + fn map(&mut self, m: &ir::Map) -> Result { if let Some((_, builder)) = self.current_map.as_mut() { for i in 0..m.len { let bt = BlockTime { diff --git a/src/thin/xml.rs b/src/thin/xml.rs index a5c35a9..42bfe0d 100644 --- a/src/thin/xml.rs +++ b/src/thin/xml.rs @@ -4,63 +4,11 @@ use std::{io::prelude::*, io::BufReader, io::Write}; use quick_xml::events::{BytesEnd, BytesStart, Event}; use quick_xml::{Reader, Writer}; +use crate::thin::ir::*; use crate::xml::*; //--------------------------------------- -#[derive(Clone)] -pub struct Superblock { - pub uuid: String, - pub time: u64, - pub transaction: u64, - pub flags: Option, - pub version: Option, - pub data_block_size: u32, - pub nr_data_blocks: u64, - pub metadata_snap: Option, -} - -#[derive(Clone)] -pub struct Device { - pub dev_id: u32, - pub mapped_blocks: u64, - pub transaction: u64, - pub creation_time: u64, - pub snap_time: u64, -} - -#[derive(Clone)] -pub struct Map { - pub thin_begin: u64, - pub data_begin: u64, - pub time: u32, - pub len: u64, -} - -#[derive(Clone)] -pub enum Visit { - Continue, - Stop, -} - -pub trait MetadataVisitor { - fn superblock_b(&mut self, sb: &Superblock) -> Result; - fn superblock_e(&mut self) -> Result; - - // Defines a shared sub tree. May only contain a 'map' (no 'ref' allowed). - fn def_shared_b(&mut self, name: &str) -> Result; - fn def_shared_e(&mut self) -> Result; - - // A device contains a number of 'map' or 'ref' items. - fn device_b(&mut self, d: &Device) -> Result; - fn device_e(&mut self) -> Result; - - fn map(&mut self, m: &Map) -> Result; - fn ref_shared(&mut self, name: &str) -> Result; - - fn eof(&mut self) -> Result; -} - pub struct XmlWriter { w: Writer, } @@ -178,7 +126,7 @@ impl MetadataVisitor for XmlWriter { fn parse_superblock(e: &BytesStart) -> Result { let mut uuid: Option = None; - let mut time: Option = None; + let mut time: Option = None; let mut transaction: Option = None; let mut flags: Option = None; let mut version: Option = None; @@ -190,7 +138,7 @@ fn parse_superblock(e: &BytesStart) -> Result { let kv = a.unwrap(); match kv.key { b"uuid" => uuid = Some(string_val(&kv)), - b"time" => time = Some(u64_val(&kv)?), + b"time" => time = Some(u32_val(&kv)?), b"transaction" => transaction = Some(u64_val(&kv)?), b"flags" => flags = Some(u32_val(&kv)?), b"version" => version = Some(u32_val(&kv)?), @@ -235,8 +183,8 @@ fn parse_device(e: &BytesStart) -> Result { let mut dev_id: Option = None; let mut mapped_blocks: Option = None; let mut transaction: Option = None; - let mut creation_time: Option = None; - let mut snap_time: Option = None; + let mut creation_time: Option = None; + let mut snap_time: Option = None; for a in e.attributes() { let kv = a.unwrap(); @@ -244,8 +192,8 @@ fn parse_device(e: &BytesStart) -> Result { b"dev_id" => dev_id = Some(u32_val(&kv)?), b"mapped_blocks" => mapped_blocks = Some(u64_val(&kv)?), b"transaction" => transaction = Some(u64_val(&kv)?), - b"creation_time" => creation_time = Some(u64_val(&kv)?), - b"snap_time" => snap_time = Some(u64_val(&kv)?), + b"creation_time" => creation_time = Some(u32_val(&kv)?), + b"snap_time" => snap_time = Some(u32_val(&kv)?), _ => return bad_attr("device", kv.key), } } diff --git a/tests/common/cache_xml_generator.rs b/tests/common/cache_xml_generator.rs index 3621d50..e7df242 100644 --- a/tests/common/cache_xml_generator.rs +++ b/tests/common/cache_xml_generator.rs @@ -3,12 +3,13 @@ use rand::prelude::*; use std::collections::HashSet; use std::fs::OpenOptions; use std::path::Path; +use thinp::cache::ir::{self, MetadataVisitor}; use thinp::cache::xml; //------------------------------------------ pub trait XmlGen { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()>; } pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { @@ -50,8 +51,8 @@ impl CacheGen { } impl XmlGen for CacheGen { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.superblock_b(&xml::Superblock { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> { + v.superblock_b(&ir::Superblock { uuid: "".to_string(), block_size: self.block_size, nr_cache_blocks: self.nr_cache_blocks, @@ -77,7 +78,7 @@ impl XmlGen for CacheGen { used.insert(oblock); // FIXME: dirty should vary - v.mapping(&xml::Map { + v.mapping(&ir::Map { cblock: cblocks[n as usize], oblock, dirty: false, diff --git a/tests/common/thin_xml_generator.rs b/tests/common/thin_xml_generator.rs index 83084d1..9d56856 100644 --- a/tests/common/thin_xml_generator.rs +++ b/tests/common/thin_xml_generator.rs @@ -4,12 +4,13 @@ use std::collections::VecDeque; use std::fs::OpenOptions; use std::ops::Range; use std::path::Path; +use thinp::thin::ir::{self, MetadataVisitor}; use thinp::thin::xml; //------------------------------------------ pub trait XmlGen { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()>; } pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { @@ -24,8 +25,8 @@ pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { g.generate_xml(&mut w) } -fn common_sb(nr_blocks: u64) -> xml::Superblock { - xml::Superblock { +fn common_sb(nr_blocks: u64) -> ir::Superblock { + ir::Superblock { uuid: "".to_string(), time: 0, transaction: 0, @@ -42,7 +43,7 @@ fn common_sb(nr_blocks: u64) -> xml::Superblock { pub struct EmptyPoolS {} impl XmlGen for EmptyPoolS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> { v.superblock_b(&common_sb(1024))?; v.superblock_e()?; Ok(()) @@ -70,16 +71,16 @@ impl SingleThinS { } impl XmlGen for SingleThinS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> { v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - v.device_b(&xml::Device { + v.device_b(&ir::Device { dev_id: 0, mapped_blocks: self.len, transaction: 0, creation_time: 0, snap_time: 0, })?; - v.map(&xml::Map { + v.map(&ir::Map { thin_begin: 0, data_begin: self.offset, time: 0, @@ -146,7 +147,7 @@ fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range) -> Vec Result<()> { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> { // Allocate each thin fully, in runs between 1 and 16. let mut runs = Vec::new(); for thin in 0..self.nr_thins { @@ -188,7 +189,7 @@ impl XmlGen for FragmentedS { // write the xml v.superblock_b(&common_sb(self.old_nr_data_blocks))?; for thin in 0..self.nr_thins { - v.device_b(&xml::Device { + v.device_b(&ir::Device { dev_id: thin, mapped_blocks: self.thin_size, transaction: 0, @@ -201,7 +202,7 @@ impl XmlGen for FragmentedS { continue; } - v.map(&xml::Map { + v.map(&ir::Map { thin_begin: m.thin_begin, data_begin: m.data_begin, time: 0, @@ -329,8 +330,8 @@ struct ThinDev { } impl ThinDev { - fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.device_b(&xml::Device { + fn emit(&self, v: &mut dyn MetadataVisitor) -> Result<()> { + v.device_b(&ir::Device { dev_id: self.thin_id, mapped_blocks: self.dev_size, transaction: 0, @@ -342,7 +343,7 @@ impl ThinDev { for r in &self.runs { match r { Run::Mapped { data_begin, len } => { - v.map(&xml::Map { + v.map(&ir::Map { thin_begin: b, data_begin: *data_begin, time: 0, @@ -522,7 +523,7 @@ impl SnapS { } impl XmlGen for SnapS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> { let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512); let origin = mk_origin(0, self.len, &mut allocator)?; diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index e796dfd..bbe78d4 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -6,7 +6,8 @@ use std::io::{Cursor, Read, Seek, SeekFrom, Write}; use std::path::Path; use thinp::file_utils; -use thinp::thin::xml::{self, Visit}; +use thinp::thin::ir::{self, MetadataVisitor, Visit}; +use thinp::thin::xml; mod common; use common::test_dir::*; @@ -91,8 +92,8 @@ struct ThinXmlVisitor<'a, V: ThinVisitor> { thin_id: Option, } -impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> { - fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { +impl<'a, V: ThinVisitor> MetadataVisitor for ThinXmlVisitor<'a, V> { + fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { self.block_size = Some(sb.data_block_size); Ok(Visit::Continue) } @@ -109,7 +110,7 @@ impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> { todo!(); } - fn device_b(&mut self, d: &xml::Device) -> Result { + fn device_b(&mut self, d: &ir::Device) -> Result { self.thin_id = Some(d.dev_id); Ok(Visit::Continue) } @@ -118,7 +119,7 @@ impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> { Ok(Visit::Continue) } - fn map(&mut self, m: &xml::Map) -> Result { + fn map(&mut self, m: &ir::Map) -> Result { for i in 0..m.len { let block = ThinBlock { thin_id: self.thin_id.unwrap(), From 56d7da66aa86c1baee7b942d789fd0ccf4eeb466 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 14 Jul 2021 23:32:39 +0800 Subject: [PATCH 2/8] [thin_dump (rust)] Tidy up - Do not use shared internals as Defs - Build Defs with common leaf sequences only, that could forms more common Defs between partially shared subtrees. --- src/thin/dump.rs | 151 ++++++++++++----------------------------------- 1 file changed, 39 insertions(+), 112 deletions(-) diff --git a/src/thin/dump.rs b/src/thin/dump.rs index 7296dfb..6f4abed 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -3,7 +3,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::io::BufWriter; use std::io::Write; -use std::ops::DerefMut; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -240,29 +239,17 @@ impl LeafVisitor for CollectLeaves { } fn collect_leaves( - ctx: &Context, + engine: Arc, roots: &BTreeSet, - mut sm: Box, ) -> Result>> { let mut map: BTreeMap> = BTreeMap::new(); + let mut sm = RestrictedSpaceMap::new(engine.get_nr_blocks()); - ctx.report - .set_title(&format!("Collecting leaves for {} roots", roots.len())); - - // FIXME: we don't want any leaves in shared. - for r in roots.iter() { - let old_count = sm.get(*r).expect("couldn't get count from space map."); - sm.set(*r, 0).expect("couldn't set count in space map."); - - let mut w = LeafWalker::new(ctx.engine.clone(), sm.deref_mut(), false); + for r in roots { + let mut w = LeafWalker::new(engine.clone(), &mut sm, false); let mut v = CollectLeaves::new(); - let mut path = vec![0]; - - // ctx.report.set_title(&format!("collecting {}", *r)); w.walk::(&mut path, &mut v, *r)?; - sm.set(*r, old_count) - .expect("couldn't set count in space map."); map.insert(*r, v.leaves); } @@ -272,59 +259,9 @@ fn collect_leaves( //------------------------------------------ -#[allow(dead_code)] -fn find_shared_nodes( - ctx: &Context, - roots: &BTreeMap, u64)>, -) -> Result<(BTreeSet, Box)> { - let nr_metadata_blocks = ctx.engine.get_nr_blocks(); - let mut sm = core_sm_without_mutex(nr_metadata_blocks, roots.len() as u32); - let mut v = NoopLeafVisitor {}; - let mut w = LeafWalker::new(ctx.engine.clone(), sm.deref_mut(), false); - - for (thin_id, (path, root)) in roots { - let mut path = path.clone(); - ctx.report.info(&format!("scanning {}", thin_id)); - w.walk::(&mut path, &mut v, *root)?; - } - - // We have to get the leaves so w is consumed and the &mut on sm - // is dropped. - let _leaves = w.get_leaves(); - let mut shared = BTreeSet::new(); - { - for i in 0..sm.get_nr_blocks().unwrap() { - if sm.get(i).expect("couldn't get count from space map.") > 1 { - shared.insert(i); - } - } - } - - /* - // FIXME: why?!! - // we're not interested in leaves (roots will get re-added later). - { - for i in 0..leaves.len() { - if leaves.contains(i) { - shared.remove(&(i as u64)); - } - } - } - */ - - Ok((shared, sm)) -} - -//------------------------------------------ - fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { let report = &ctx.report; let engine = &ctx.engine; - - // superblock - report.set_title("Reading superblock"); - //let metadata_root = unpack::(&sb.metadata_sm_root[0..])?; - //let data_root = unpack::(&sb.data_sm_root[0..])?; let mut path = vec![0]; report.set_title("Reading device details"); @@ -338,17 +275,15 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { btree_to_map_with_path::(&mut path, engine.clone(), sm, true, sb.mapping_root)?; } - let sm = Box::new(RestrictedSpaceMap::new(engine.get_nr_blocks())); + report.set_title(&format!("Collecting leaves for {} roots", roots.len())); let mapping_roots = roots.values().map(|(_, root)| *root).collect(); - let entry_map = collect_leaves(&ctx, &mapping_roots, sm)?; + let entry_map = collect_leaves(engine.clone(), &mapping_roots)?; + let defs = Vec::new(); let mut devs = Vec::new(); - - let mut seen = BTreeSet::new(); for (thin_id, (_path, root)) in roots { let id = thin_id as u64; let detail = details.get(&id).expect("couldn't find device details"); - seen.insert(root); let es = entry_map.get(&root).unwrap(); let kr = KeyRange::new(); // FIXME: finish devs.push(Device { @@ -366,7 +301,6 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { //------------------------------------------ -#[allow(dead_code)] fn gather_entries(g: &mut Gatherer, es: &[Entry]) { g.new_seq(); for e in es { @@ -381,7 +315,22 @@ fn gather_entries(g: &mut Gatherer, es: &[Entry]) { } } -#[allow(dead_code)] +fn build_runs(devs: &[Device]) -> BTreeMap> { + let mut g = Gatherer::new(); + + for d in devs { + gather_entries(&mut g, &d.map.entries); + } + + // The runs become defs that just contain leaves. + let mut runs = BTreeMap::new(); + for run in g.gather() { + runs.insert(run[0], run); + } + + runs +} + fn entries_to_runs(runs: &BTreeMap>, es: &[Entry]) -> Vec { use Entry::*; @@ -408,51 +357,28 @@ fn entries_to_runs(runs: &BTreeMap>, es: &[Entry]) -> Vec { result } -// FIXME: do we really need to track kr? -// FIXME: I think this may be better done as part of restore. -#[allow(dead_code)] -fn optimise_metadata(md: Metadata) -> Result { - use Entry::*; - - let mut g = Gatherer::new(); - for d in &md.defs { - gather_entries(&mut g, &d.map.entries); - } - - for d in &md.devs { - gather_entries(&mut g, &d.map.entries); - } - +fn build_defs(runs: BTreeMap>) -> Vec { let mut defs = Vec::new(); - let mut devs = Vec::new(); - let mut runs = BTreeMap::new(); - for run in g.gather() { - runs.insert(run[0], run); - } - eprintln!("{} runs", runs.len()); - - // The runs become additional defs that just contain leaves. for (head, run) in runs.iter() { let kr = KeyRange::new(); - let entries: Vec = run.iter().map(|b| Leaf(*b)).collect(); + let entries: Vec = run.iter().map(|b| Entry::Leaf(*b)).collect(); defs.push(Def { def_id: *head, map: Mapping { kr, entries }, }); } - // Expand old defs to use the new atomic runs - for d in &md.defs { - let kr = KeyRange::new(); - let entries = entries_to_runs(&runs, &d.map.entries); + defs +} - defs.push(Def { - def_id: d.def_id, - map: Mapping { kr, entries }, - }); - } +// FIXME: do we really need to track kr? +// FIXME: I think this may be better done as part of restore. +fn optimise_metadata(md: Metadata) -> Result { + let runs = build_runs(&md.devs); + eprintln!("{} runs", runs.len()); // Expand old devs to use the new atomic runs + let mut devs = Vec::new(); for d in &md.devs { let kr = KeyRange::new(); let entries = entries_to_runs(&runs, &d.map.entries); @@ -463,6 +389,8 @@ fn optimise_metadata(md: Metadata) -> Result { }); } + let defs = build_defs(runs); + Ok(Metadata { defs, devs }) } @@ -606,17 +534,16 @@ pub fn dump(opts: ThinDumpOptions) -> Result<()> { let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; let md = build_metadata(&ctx, &sb)?; + ctx.report + .set_title("Optimising metadata to improve leaf packing"); + let md = optimise_metadata(md)?; + let mut writer: Box; if opts.output.is_some() { writer = Box::new(BufWriter::new(File::create(opts.output.unwrap())?)); } else { writer = Box::new(BufWriter::new(std::io::stdout())); } - - ctx.report - .set_title("Optimising metadata to improve leaf packing"); - let md = optimise_metadata(md)?; - dump_metadata(&ctx, &mut writer, &sb, &md) } From d42bde371f299103a36522d46e3e2bdc6ad8119b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 15 Jul 2021 22:47:58 +0800 Subject: [PATCH 3/8] [thin_dump (rust)] Prepare for thin_repair - Adapt function interfaces for repairing purpose - Pull out reuseable structures --- src/thin/dump.rs | 268 ++++--------------------------------------- src/thin/metadata.rs | 236 +++++++++++++++++++++++++++++++++++++ src/thin/mod.rs | 1 + 3 files changed, 262 insertions(+), 243 deletions(-) create mode 100644 src/thin/metadata.rs diff --git a/src/thin/dump.rs b/src/thin/dump.rs index 6f4abed..f588c5d 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::io::BufWriter; use std::io::Write; @@ -9,16 +8,13 @@ use std::sync::{Arc, Mutex}; use crate::checksum; use crate::io_engine::{AsyncIoEngine, Block, IoEngine, SyncIoEngine}; use crate::pdata::btree::{self, *}; -use crate::pdata::btree_leaf_walker::*; use crate::pdata::btree_walker::*; -use crate::pdata::space_map::*; use crate::pdata::space_map_common::*; use crate::pdata::unpack::*; use crate::report::*; use crate::thin::block_time::*; -use crate::thin::device_detail::*; use crate::thin::ir::{self, MetadataVisitor}; -use crate::thin::runs::*; +use crate::thin::metadata::*; use crate::thin::superblock::*; use crate::thin::xml; @@ -176,226 +172,6 @@ fn mk_context(opts: &ThinDumpOptions) -> Result { //------------------------------------------ -type DefId = u64; -type ThinId = u32; - -#[derive(Clone)] -enum Entry { - Leaf(u64), - Ref(DefId), -} - -#[derive(Clone)] -struct Mapping { - kr: KeyRange, - entries: Vec, -} - -#[derive(Clone)] -struct Device { - thin_id: ThinId, - detail: DeviceDetail, - map: Mapping, -} - -#[derive(Clone)] -struct Def { - def_id: DefId, - map: Mapping, -} - -#[derive(Clone)] -struct Metadata { - defs: Vec, - devs: Vec, -} - -//------------------------------------------ - -struct CollectLeaves { - leaves: Vec, -} - -impl CollectLeaves { - fn new() -> CollectLeaves { - CollectLeaves { leaves: Vec::new() } - } -} - -impl LeafVisitor for CollectLeaves { - fn visit(&mut self, _kr: &KeyRange, b: u64) -> btree::Result<()> { - self.leaves.push(Entry::Leaf(b)); - Ok(()) - } - - fn visit_again(&mut self, b: u64) -> btree::Result<()> { - self.leaves.push(Entry::Ref(b)); - Ok(()) - } - - fn end_walk(&mut self) -> btree::Result<()> { - Ok(()) - } -} - -fn collect_leaves( - engine: Arc, - roots: &BTreeSet, -) -> Result>> { - let mut map: BTreeMap> = BTreeMap::new(); - let mut sm = RestrictedSpaceMap::new(engine.get_nr_blocks()); - - for r in roots { - let mut w = LeafWalker::new(engine.clone(), &mut sm, false); - let mut v = CollectLeaves::new(); - let mut path = vec![0]; - w.walk::(&mut path, &mut v, *r)?; - - map.insert(*r, v.leaves); - } - - Ok(map) -} - -//------------------------------------------ - -fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { - let report = &ctx.report; - let engine = &ctx.engine; - let mut path = vec![0]; - - report.set_title("Reading device details"); - let details = btree_to_map::(&mut path, engine.clone(), true, sb.details_root)?; - - report.set_title("Reading mappings roots"); - let roots; - { - let sm = Arc::new(Mutex::new(RestrictedSpaceMap::new(engine.get_nr_blocks()))); - roots = - btree_to_map_with_path::(&mut path, engine.clone(), sm, true, sb.mapping_root)?; - } - - report.set_title(&format!("Collecting leaves for {} roots", roots.len())); - let mapping_roots = roots.values().map(|(_, root)| *root).collect(); - let entry_map = collect_leaves(engine.clone(), &mapping_roots)?; - - let defs = Vec::new(); - let mut devs = Vec::new(); - for (thin_id, (_path, root)) in roots { - let id = thin_id as u64; - let detail = details.get(&id).expect("couldn't find device details"); - let es = entry_map.get(&root).unwrap(); - let kr = KeyRange::new(); // FIXME: finish - devs.push(Device { - thin_id: thin_id as u32, - detail: *detail, - map: Mapping { - kr, - entries: es.to_vec(), - }, - }); - } - - Ok(Metadata { defs, devs }) -} - -//------------------------------------------ - -fn gather_entries(g: &mut Gatherer, es: &[Entry]) { - g.new_seq(); - for e in es { - match e { - Entry::Leaf(b) => { - g.next(*b); - } - Entry::Ref(_id) => { - g.new_seq(); - } - } - } -} - -fn build_runs(devs: &[Device]) -> BTreeMap> { - let mut g = Gatherer::new(); - - for d in devs { - gather_entries(&mut g, &d.map.entries); - } - - // The runs become defs that just contain leaves. - let mut runs = BTreeMap::new(); - for run in g.gather() { - runs.insert(run[0], run); - } - - runs -} - -fn entries_to_runs(runs: &BTreeMap>, es: &[Entry]) -> Vec { - use Entry::*; - - let mut result = Vec::new(); - let mut entry_index = 0; - while entry_index < es.len() { - match es[entry_index] { - Ref(id) => { - result.push(Ref(id)); - entry_index += 1; - } - Leaf(b) => { - if let Some(run) = runs.get(&b) { - result.push(Ref(b)); - entry_index += run.len(); - } else { - result.push(Leaf(b)); - entry_index += 1; - } - } - } - } - - result -} - -fn build_defs(runs: BTreeMap>) -> Vec { - let mut defs = Vec::new(); - for (head, run) in runs.iter() { - let kr = KeyRange::new(); - let entries: Vec = run.iter().map(|b| Entry::Leaf(*b)).collect(); - defs.push(Def { - def_id: *head, - map: Mapping { kr, entries }, - }); - } - - defs -} - -// FIXME: do we really need to track kr? -// FIXME: I think this may be better done as part of restore. -fn optimise_metadata(md: Metadata) -> Result { - let runs = build_runs(&md.devs); - eprintln!("{} runs", runs.len()); - - // Expand old devs to use the new atomic runs - let mut devs = Vec::new(); - for d in &md.devs { - let kr = KeyRange::new(); - let entries = entries_to_runs(&runs, &d.map.entries); - devs.push(Device { - thin_id: d.thin_id, - detail: d.detail, - map: Mapping { kr, entries }, - }); - } - - let defs = build_defs(runs); - - Ok(Metadata { defs, devs }) -} - -//------------------------------------------ - fn emit_leaf(v: &mut MappingVisitor, b: &Block) -> Result<()> { use Node::*; let path = Vec::new(); @@ -445,20 +221,20 @@ where Ok(()) } -fn emit_leaves(ctx: &Context, out: &mut dyn MetadataVisitor, ls: &[u64]) -> Result<()> { +fn emit_leaves(engine: Arc, out: &mut dyn MetadataVisitor, ls: &[u64]) -> Result<()> { let mut v = MappingVisitor::new(out); let proc = |b| { emit_leaf(&mut v, &b)?; Ok(()) }; - read_for(ctx.engine.clone(), ls, proc)?; + read_for(engine, ls, proc)?; v.end_walk().map_err(|_| anyhow!("failed to emit leaves")) } -fn emit_entries( - ctx: &Context, - out: &mut xml::XmlWriter, +fn emit_entries( + engine: Arc, + out: &mut dyn MetadataVisitor, entries: &[Entry], ) -> Result<()> { let mut leaves = Vec::new(); @@ -470,7 +246,7 @@ fn emit_entries( } Entry::Ref(id) => { if !leaves.is_empty() { - emit_leaves(&ctx, out, &leaves[0..])?; + emit_leaves(engine.clone(), out, &leaves[0..])?; leaves.clear(); } let str = format!("{}", id); @@ -480,16 +256,20 @@ fn emit_entries( } if !leaves.is_empty() { - emit_leaves(&ctx, out, &leaves[0..])?; + emit_leaves(engine, out, &leaves[0..])?; } Ok(()) } -fn dump_metadata(ctx: &Context, w: &mut dyn Write, sb: &Superblock, md: &Metadata) -> Result<()> { +pub fn dump_metadata( + engine: Arc, + out: &mut dyn MetadataVisitor, + sb: &Superblock, + md: &Metadata, +) -> Result<()> { let data_root = unpack::(&sb.data_sm_root[0..])?; - let mut out = xml::XmlWriter::new(w); - let xml_sb = ir::Superblock { + let out_sb = ir::Superblock { uuid: "".to_string(), time: sb.time, transaction: sb.transaction_id, @@ -499,16 +279,16 @@ fn dump_metadata(ctx: &Context, w: &mut dyn Write, sb: &Superblock, md: &Metadat nr_data_blocks: data_root.nr_blocks, metadata_snap: None, }; - out.superblock_b(&xml_sb)?; + out.superblock_b(&out_sb)?; - ctx.report.set_title("Dumping shared regions"); + // ctx.report.set_title("Dumping shared regions"); for d in &md.defs { out.def_shared_b(&format!("{}", d.def_id))?; - emit_entries(ctx, &mut out, &d.map.entries)?; + emit_entries(engine.clone(), out, &d.map.entries)?; out.def_shared_e()?; } - ctx.report.set_title("Dumping devices"); + // ctx.report.set_title("Dumping devices"); for dev in &md.devs { let device = ir::Device { dev_id: dev.thin_id, @@ -518,7 +298,7 @@ fn dump_metadata(ctx: &Context, w: &mut dyn Write, sb: &Superblock, md: &Metadat snap_time: dev.detail.snapshotted_time, }; out.device_b(&device)?; - emit_entries(ctx, &mut out, &dev.map.entries)?; + emit_entries(engine.clone(), out, &dev.map.entries)?; out.device_e()?; } out.superblock_e()?; @@ -532,19 +312,21 @@ fn dump_metadata(ctx: &Context, w: &mut dyn Write, sb: &Superblock, md: &Metadat pub fn dump(opts: ThinDumpOptions) -> Result<()> { let ctx = mk_context(&opts)?; let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; - let md = build_metadata(&ctx, &sb)?; + let md = build_metadata(ctx.engine.clone(), &sb)?; ctx.report .set_title("Optimising metadata to improve leaf packing"); let md = optimise_metadata(md)?; - let mut writer: Box; + let writer: Box; if opts.output.is_some() { writer = Box::new(BufWriter::new(File::create(opts.output.unwrap())?)); } else { writer = Box::new(BufWriter::new(std::io::stdout())); } - dump_metadata(&ctx, &mut writer, &sb, &md) + let mut out = xml::XmlWriter::new(writer); + + dump_metadata(ctx.engine, &mut out, &sb, &md) } //------------------------------------------ diff --git a/src/thin/metadata.rs b/src/thin/metadata.rs new file mode 100644 index 0000000..2972448 --- /dev/null +++ b/src/thin/metadata.rs @@ -0,0 +1,236 @@ +use anyhow::Result; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +use crate::io_engine::IoEngine; +use crate::pdata::btree::{self, *}; +use crate::pdata::btree_leaf_walker::*; +use crate::pdata::btree_walker::*; +use crate::pdata::space_map::*; +use crate::thin::block_time::*; +use crate::thin::device_detail::*; +use crate::thin::runs::*; +use crate::thin::superblock::*; + +//------------------------------------------ + +type DefId = u64; +type ThinId = u32; + +#[derive(Clone)] +pub enum Entry { + Leaf(u64), + Ref(DefId), +} + +#[derive(Clone)] +pub struct Mapping { + pub kr: KeyRange, + pub entries: Vec, +} + +#[derive(Clone)] +pub struct Device { + pub thin_id: ThinId, + pub detail: DeviceDetail, + pub map: Mapping, +} + +#[derive(Clone)] +pub struct Def { + pub def_id: DefId, + pub map: Mapping, +} + +#[derive(Clone)] +pub struct Metadata { + pub defs: Vec, + pub devs: Vec, +} + +//------------------------------------------ + +struct CollectLeaves { + leaves: Vec, +} + +impl CollectLeaves { + fn new() -> CollectLeaves { + CollectLeaves { leaves: Vec::new() } + } +} + +impl LeafVisitor for CollectLeaves { + fn visit(&mut self, _kr: &KeyRange, b: u64) -> btree::Result<()> { + self.leaves.push(Entry::Leaf(b)); + Ok(()) + } + + fn visit_again(&mut self, b: u64) -> btree::Result<()> { + self.leaves.push(Entry::Ref(b)); + Ok(()) + } + + fn end_walk(&mut self) -> btree::Result<()> { + Ok(()) + } +} + +fn collect_leaves( + engine: Arc, + roots: &BTreeSet, +) -> Result>> { + let mut map: BTreeMap> = BTreeMap::new(); + let mut sm = RestrictedSpaceMap::new(engine.get_nr_blocks()); + + for r in roots { + let mut w = LeafWalker::new(engine.clone(), &mut sm, false); + let mut v = CollectLeaves::new(); + let mut path = vec![0]; + w.walk::(&mut path, &mut v, *r)?; + + map.insert(*r, v.leaves); + } + + Ok(map) +} + +//------------------------------------------ + +pub fn build_metadata( + engine: Arc, + sb: &Superblock, +) -> Result { + let mut path = vec![0]; + + // report.set_title("Reading device details"); + let details = btree_to_map::(&mut path, engine.clone(), true, sb.details_root)?; + + // report.set_title("Reading mappings roots"); + let roots; + { + let sm = Arc::new(Mutex::new(RestrictedSpaceMap::new(engine.get_nr_blocks()))); + roots = + btree_to_map_with_path::(&mut path, engine.clone(), sm, true, sb.mapping_root)?; + } + + // report.set_title(&format!("Collecting leaves for {} roots", roots.len())); + let mapping_roots = roots.values().map(|(_, root)| *root).collect(); + let entry_map = collect_leaves(engine.clone(), &mapping_roots)?; + + let defs = Vec::new(); + let mut devs = Vec::new(); + for (thin_id, (_path, root)) in roots { + let id = thin_id as u64; + let detail = details.get(&id).expect("couldn't find device details"); + let es = entry_map.get(&root).unwrap(); + let kr = KeyRange::new(); // FIXME: finish + devs.push(Device { + thin_id: thin_id as u32, + detail: *detail, + map: Mapping { + kr, + entries: es.to_vec(), + }, + }); + } + + Ok(Metadata { defs, devs }) +} + +//------------------------------------------ + +fn gather_entries(g: &mut Gatherer, es: &[Entry]) { + g.new_seq(); + for e in es { + match e { + Entry::Leaf(b) => { + g.next(*b); + } + Entry::Ref(_id) => { + g.new_seq(); + } + } + } +} + +fn build_runs(devs: &[Device]) -> BTreeMap> { + let mut g = Gatherer::new(); + + for d in devs { + gather_entries(&mut g, &d.map.entries); + } + + // The runs become defs that just contain leaves. + let mut runs = BTreeMap::new(); + for run in g.gather() { + runs.insert(run[0], run); + } + + runs +} + +fn entries_to_runs(runs: &BTreeMap>, es: &[Entry]) -> Vec { + use Entry::*; + + let mut result = Vec::new(); + let mut entry_index = 0; + while entry_index < es.len() { + match es[entry_index] { + Ref(id) => { + result.push(Ref(id)); + entry_index += 1; + } + Leaf(b) => { + if let Some(run) = runs.get(&b) { + result.push(Ref(b)); + entry_index += run.len(); + } else { + result.push(Leaf(b)); + entry_index += 1; + } + } + } + } + + result +} + +fn build_defs(runs: BTreeMap>) -> Vec { + let mut defs = Vec::new(); + for (head, run) in runs.iter() { + let kr = KeyRange::new(); + let entries: Vec = run.iter().map(|b| Entry::Leaf(*b)).collect(); + defs.push(Def { + def_id: *head, + map: Mapping { kr, entries }, + }); + } + + defs +} + +// FIXME: do we really need to track kr? +// FIXME: I think this may be better done as part of restore. +pub fn optimise_metadata(md: Metadata) -> Result { + let runs = build_runs(&md.devs); + eprintln!("{} runs", runs.len()); + + // Expand old devs to use the new atomic runs + let mut devs = Vec::new(); + for d in &md.devs { + let kr = KeyRange::new(); + let entries = entries_to_runs(&runs, &d.map.entries); + devs.push(Device { + thin_id: d.thin_id, + detail: d.detail, + map: Mapping { kr, entries }, + }); + } + + let defs = build_defs(runs); + + Ok(Metadata { defs, devs }) +} + +//------------------------------------------ diff --git a/src/thin/mod.rs b/src/thin/mod.rs index cbaa118..101c9c2 100644 --- a/src/thin/mod.rs +++ b/src/thin/mod.rs @@ -3,6 +3,7 @@ pub mod check; pub mod device_detail; pub mod dump; pub mod ir; +pub mod metadata; pub mod restore; pub mod runs; pub mod superblock; From dccd84471481bc6e97ed1036fa60b4cf3db611b9 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 16 Jul 2021 17:46:05 +0800 Subject: [PATCH 4/8] [thin_restore (rust)] Prepare for thin_repair - Finalize the metadata in the Restorer - Make the Restorer public --- src/bin/thin_restore.rs | 8 ++- src/thin/restore.rs | 112 ++++++++++++++++++++-------------------- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/src/bin/thin_restore.rs b/src/bin/thin_restore.rs index 093f376..34cfb85 100644 --- a/src/bin/thin_restore.rs +++ b/src/bin/thin_restore.rs @@ -22,6 +22,12 @@ fn main() { .long("async-io") .hidden(true), ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) // options .arg( Arg::with_name("INPUT") @@ -33,7 +39,7 @@ fn main() { ) .arg( Arg::with_name("OUTPUT") - .help("Specify the output device to check") + .help("Specify the output device") .short("o") .long("output") .value_name("OUTPUT") diff --git a/src/thin/restore.rs b/src/thin/restore.rs index cab683a..e6a756f 100644 --- a/src/thin/restore.rs +++ b/src/thin/restore.rs @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex}; use crate::io_engine::*; use crate::pdata::btree_builder::*; use crate::pdata::space_map::*; +use crate::pdata::space_map_common::SMRoot; use crate::pdata::space_map_disk::*; use crate::pdata::space_map_metadata::*; use crate::pdata::unpack::Pack; @@ -58,13 +59,7 @@ impl std::fmt::Display for MappedSection { //------------------------------------------ -struct RestoreResult { - sb: ir::Superblock, - devices: BTreeMap, - data_sm: Arc>, -} - -struct Restorer<'a> { +pub struct Restorer<'a> { w: &'a mut WriteBatcher, report: Arc, @@ -81,7 +76,7 @@ struct Restorer<'a> { } impl<'a> Restorer<'a> { - fn new(w: &'a mut WriteBatcher, report: Arc) -> Self { + pub fn new(w: &'a mut WriteBatcher, report: Arc) -> Self { Restorer { w, report, @@ -94,17 +89,6 @@ impl<'a> Restorer<'a> { } } - fn get_result(self) -> Result { - if self.sb.is_none() { - return Err(anyhow!("No superblock found in xml file")); - } - Ok(RestoreResult { - sb: self.sb.unwrap(), - devices: self.devices, - data_sm: self.data_sm.unwrap(), - }) - } - fn begin_section(&mut self, section: MappedSection) -> Result { if let Some((outer, _)) = self.current_map.as_ref() { let msg = format!( @@ -134,17 +118,65 @@ impl<'a> Restorer<'a> { Err(anyhow!(msg)) } } + + // Build the device details and the top level mapping trees + fn build_device_details(&mut self) -> Result<(u64, u64)> { + let mut details_builder: BTreeBuilder = + BTreeBuilder::new(Box::new(NoopRC {})); + let mut dev_builder: BTreeBuilder = BTreeBuilder::new(Box::new(NoopRC {})); + for (thin_id, (detail, root)) in self.devices.iter() { + details_builder.push_value(self.w, *thin_id as u64, *detail)?; + dev_builder.push_value(self.w, *thin_id as u64, *root)?; + } + let details_root = details_builder.complete(self.w)?; + let mapping_root = dev_builder.complete(self.w)?; + + Ok((details_root, mapping_root)) + } + + fn finalize(&mut self) -> Result<()> { + let (details_root, mapping_root) = self.build_device_details()?; + + // Build data space map + let data_sm = self.data_sm.as_ref().unwrap(); + let data_sm_root = build_data_sm(self.w, data_sm.lock().unwrap().deref())?; + + // Build metadata space map + let (metadata_sm, metadata_sm_root) = build_metadata_sm(self.w)?; + + // Write the superblock + let sb = self.sb.as_ref().unwrap(); + let sb = superblock::Superblock { + flags: SuperblockFlags { needs_check: false }, + block: SUPERBLOCK_LOCATION, + version: 2, + time: sb.time as u32, + transaction_id: sb.transaction, + metadata_snap: 0, + data_sm_root, + metadata_sm_root, + mapping_root, + details_root, + data_block_size: sb.data_block_size, + nr_metadata_blocks: metadata_sm.nr_blocks, + }; + write_superblock(self.w.engine.as_ref(), SUPERBLOCK_LOCATION, &sb) + } } impl<'a> MetadataVisitor for Restorer<'a> { fn superblock_b(&mut self, sb: &ir::Superblock) -> Result { self.sb = Some(sb.clone()); self.data_sm = Some(core_sm(sb.nr_data_blocks, u32::MAX)); - self.w.alloc()?; + let b = self.w.alloc()?; + if b.loc != SUPERBLOCK_LOCATION { + return Err(anyhow!("superblock was occupied")); + } Ok(Visit::Continue) } fn superblock_e(&mut self) -> Result { + self.finalize()?; Ok(Visit::Continue) } @@ -249,13 +281,13 @@ fn build_data_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result> { /// Writes the metadata space map to disk. Returns the space map root that needs /// to be written to the superblock. -fn build_metadata_sm(w: &mut WriteBatcher) -> Result> { +fn build_metadata_sm(w: &mut WriteBatcher) -> Result<(SMRoot, Vec)> { let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; let mut cur = Cursor::new(&mut sm_root); let r = write_metadata_sm(w)?; r.pack(&mut cur)?; - Ok(sm_root) + Ok((r, sm_root)) } //------------------------------------------ @@ -303,42 +335,8 @@ pub fn restore(opts: ThinRestoreOptions) -> Result<()> { let sm = core_sm(ctx.engine.get_nr_blocks(), max_count); let mut w = WriteBatcher::new(ctx.engine.clone(), sm.clone(), ctx.engine.get_batch_size()); - let mut restorer = Restorer::new(&mut w, ctx.report.clone()); + let mut restorer = Restorer::new(&mut w, ctx.report); xml::read(input, &mut restorer)?; - let result = restorer.get_result()?; - - // Build the device details and top level mapping tree - let mut details_builder: BTreeBuilder = BTreeBuilder::new(Box::new(NoopRC {})); - let mut dev_builder: BTreeBuilder = BTreeBuilder::new(Box::new(NoopRC {})); - for (thin_id, (detail, root)) in &result.devices { - details_builder.push_value(&mut w, *thin_id as u64, *detail)?; - dev_builder.push_value(&mut w, *thin_id as u64, *root)?; - } - let details_root = details_builder.complete(&mut w)?; - let mapping_root = dev_builder.complete(&mut w)?; - - // Build data space map - let data_sm_root = build_data_sm(&mut w, result.data_sm.lock().unwrap().deref())?; - - // Build metadata space map - let metadata_sm_root = build_metadata_sm(&mut w)?; - - // Write the superblock - let sb = superblock::Superblock { - flags: SuperblockFlags { needs_check: false }, - block: SUPERBLOCK_LOCATION, - version: 2, - time: result.sb.time as u32, - transaction_id: result.sb.transaction, - metadata_snap: 0, - data_sm_root, - metadata_sm_root, - mapping_root, - details_root, - data_block_size: result.sb.data_block_size, - nr_metadata_blocks: ctx.engine.get_nr_blocks(), - }; - write_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?; Ok(()) } From 85581cdf5a47d5856fd6d1c78c5afd9faf1c7dd0 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 16 Jul 2021 20:13:20 +0800 Subject: [PATCH 5/8] [thin_repair (rust)] First pass at thin_repair --- src/bin/thin_repair.rs | 86 ++++++++++++++++++++++++++++++++++++++++++ src/thin/mod.rs | 1 + src/thin/repair.rs | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 src/bin/thin_repair.rs create mode 100644 src/thin/repair.rs diff --git a/src/bin/thin_repair.rs b/src/bin/thin_repair.rs new file mode 100644 index 0000000..7d41bf3 --- /dev/null +++ b/src/bin/thin_repair.rs @@ -0,0 +1,86 @@ +extern crate clap; +extern crate thinp; + +use atty::Stream; +use clap::{App, Arg}; +use std::path::Path; +use std::process; +use std::process::exit; +use std::sync::Arc; +use thinp::file_utils; +use thinp::report::*; +use thinp::thin::repair::{repair, ThinRepairOptions}; + +fn main() { + let parser = App::new("thin_repair") + .version(thinp::version::tools_version()) + .about("Repair thin-provisioning metadata, and write it to different device or file") + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) + // options + .arg( + Arg::with_name("INPUT") + .help("Specify the input device") + .short("i") + .long("input") + .value_name("INPUT") + .required(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output device") + .short("o") + .long("output") + .value_name("OUTPUT") + .required(true), + ) + .arg( + Arg::with_name("OVERRIDE_MAPPING_ROOT") + .help("Specify a mapping root to use") + .long("override-mapping-root") + .value_name("OVERRIDE_MAPPING_ROOT") + .takes_value(true), + ); + + let matches = parser.get_matches(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); + + if !file_utils::file_exists(input_file) { + eprintln!("Couldn't find input file '{:?}'.", &input_file); + exit(1); + } + + let report; + + if matches.is_present("QUIET") { + report = std::sync::Arc::new(mk_quiet_report()); + } else if atty::is(Stream::Stdout) { + report = std::sync::Arc::new(mk_progress_bar_report()); + } else { + report = Arc::new(mk_simple_report()); + } + + let opts = ThinRepairOptions { + input: &input_file, + output: &output_file, + async_io: matches.is_present("ASYNC_IO"), + report, + }; + + if let Err(reason) = repair(opts) { + eprintln!("{}", reason); + process::exit(1); + } +} diff --git a/src/thin/mod.rs b/src/thin/mod.rs index 101c9c2..1f44e9e 100644 --- a/src/thin/mod.rs +++ b/src/thin/mod.rs @@ -4,6 +4,7 @@ pub mod device_detail; pub mod dump; pub mod ir; pub mod metadata; +pub mod repair; pub mod restore; pub mod runs; pub mod superblock; diff --git a/src/thin/repair.rs b/src/thin/repair.rs new file mode 100644 index 0000000..89d6e09 --- /dev/null +++ b/src/thin/repair.rs @@ -0,0 +1,71 @@ +use anyhow::Result; +use std::path::Path; +use std::sync::Arc; + +use crate::io_engine::*; +use crate::pdata::space_map::*; +use crate::report::*; +use crate::thin::dump::*; +use crate::thin::metadata::*; +use crate::thin::restore::*; +use crate::thin::superblock::*; +use crate::write_batcher::*; + +//------------------------------------------ + +pub struct ThinRepairOptions<'a> { + pub input: &'a Path, + pub output: &'a Path, + pub async_io: bool, + pub report: Arc, +} + +struct Context { + report: Arc, + engine_in: Arc, + engine_out: Arc, +} + +const MAX_CONCURRENT_IO: u32 = 1024; + +fn new_context(opts: &ThinRepairOptions) -> Result { + let engine_in: Arc; + let engine_out: Arc; + + if opts.async_io { + engine_in = Arc::new(AsyncIoEngine::new(opts.input, MAX_CONCURRENT_IO, true)?); + engine_out = Arc::new(AsyncIoEngine::new(opts.output, MAX_CONCURRENT_IO, true)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine_in = Arc::new(SyncIoEngine::new(opts.input, nr_threads, true)?); + engine_out = Arc::new(SyncIoEngine::new(opts.output, nr_threads, true)?); + } + + Ok(Context { + report: opts.report.clone(), + engine_in, + engine_out, + }) +} + +//------------------------------------------ + +pub fn repair(opts: ThinRepairOptions) -> Result<()> { + let ctx = new_context(&opts)?; + + let sb = read_superblock(ctx.engine_in.as_ref(), SUPERBLOCK_LOCATION)?; + let md = build_metadata(ctx.engine_in.clone(), &sb)?; + let md = optimise_metadata(md)?; + + let sm = core_sm(ctx.engine_out.get_nr_blocks(), u32::MAX); + let mut w = WriteBatcher::new( + ctx.engine_out.clone(), + sm.clone(), + ctx.engine_out.get_batch_size(), + ); + let mut restorer = Restorer::new(&mut w, ctx.report); + + dump_metadata(ctx.engine_in, &mut restorer, &sb, &md) +} + +//------------------------------------------ From 4b9766846e736581781676b8937bc1da778cafe8 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Mon, 19 Jul 2021 10:44:33 +0800 Subject: [PATCH 6/8] [cache (rust)] Prepare for cache_repair - Adapt function interfaces for repairing purpose - Finalize the metadata in the Restorer - Make the Restorer public --- src/bin/cache_restore.rs | 6 +++ src/cache/dump.rs | 23 ++++------ src/cache/restore.rs | 98 ++++++++++++++++------------------------ 3 files changed, 55 insertions(+), 72 deletions(-) diff --git a/src/bin/cache_restore.rs b/src/bin/cache_restore.rs index e1f4209..f580777 100644 --- a/src/bin/cache_restore.rs +++ b/src/bin/cache_restore.rs @@ -29,6 +29,12 @@ fn main() { .value_name("OVERRIDE_MAPPING_ROOT") .takes_value(true), ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) // options .arg( Arg::with_name("INPUT") diff --git a/src/cache/dump.rs b/src/cache/dump.rs index c5c33c1..a5b9309 100644 --- a/src/cache/dump.rs +++ b/src/cache/dump.rs @@ -218,15 +218,12 @@ fn mk_context(opts: &CacheDumpOptions) -> anyhow::Result { Ok(Context { engine }) } -fn dump_metadata( - ctx: &Context, - w: &mut dyn Write, +pub fn dump_metadata( + engine: Arc, + out: &mut dyn MetadataVisitor, sb: &Superblock, _repair: bool, ) -> anyhow::Result<()> { - let engine = &ctx.engine; - - let mut out = xml::XmlWriter::new(w); let xml_sb = ir::Superblock { uuid: "".to_string(), block_size: sb.data_block_size, @@ -240,7 +237,7 @@ fn dump_metadata( let valid_mappings = match sb.version { 1 => { let w = ArrayWalker::new(engine.clone(), false); - let mut emitter = format1::MappingEmitter::new(sb.cache_blocks as usize, &mut out); + let mut emitter = format1::MappingEmitter::new(sb.cache_blocks as usize, out); w.walk(&mut emitter, sb.mapping_root)?; emitter.get_valid() } @@ -263,7 +260,7 @@ fn dump_metadata( let w = ArrayWalker::new(engine.clone(), false); let mut emitter = - format2::MappingEmitter::new(sb.cache_blocks as usize, dirty_bits, &mut out); + format2::MappingEmitter::new(sb.cache_blocks as usize, dirty_bits, out); w.walk(&mut emitter, sb.mapping_root)?; emitter.get_valid() } @@ -276,7 +273,7 @@ fn dump_metadata( out.hints_b()?; { let w = ArrayWalker::new(engine.clone(), false); - let mut emitter = HintEmitter::new(&mut out, valid_mappings); + let mut emitter = HintEmitter::new(out, valid_mappings); w.walk(&mut emitter, sb.hint_root)?; } out.hints_e()?; @@ -289,17 +286,17 @@ fn dump_metadata( pub fn dump(opts: CacheDumpOptions) -> anyhow::Result<()> { let ctx = mk_context(&opts)?; - let engine = &ctx.engine; - let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; + let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?; - let mut writer: Box; + let writer: Box; if opts.output.is_some() { writer = Box::new(BufWriter::new(File::create(opts.output.unwrap())?)); } else { writer = Box::new(BufWriter::new(std::io::stdout())); } + let mut out = xml::XmlWriter::new(writer); - dump_metadata(&ctx, &mut writer, &sb, opts.repair) + dump_metadata(ctx.engine.clone(), &mut out, &sb, opts.repair) } //------------------------------------------ diff --git a/src/cache/restore.rs b/src/cache/restore.rs index 5d5f23c..067462a 100644 --- a/src/cache/restore.rs +++ b/src/cache/restore.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Result}; +use anyhow::Result; use std::convert::TryInto; use std::fs::OpenOptions; @@ -56,15 +56,7 @@ fn mk_context(opts: &CacheRestoreOptions) -> anyhow::Result { //------------------------------------------ -struct RestoreResult { - sb: ir::Superblock, - mapping_root: u64, - dirty_root: Option, - hint_root: u64, - discard_root: u64, -} - -struct Restorer<'a> { +pub struct Restorer<'a> { write_batcher: &'a mut WriteBatcher, sb: Option, mapping_builder: Option>, @@ -78,7 +70,7 @@ struct Restorer<'a> { } impl<'a> Restorer<'a> { - fn new(w: &'a mut WriteBatcher) -> Restorer<'a> { + pub fn new(w: &'a mut WriteBatcher) -> Restorer<'a> { Restorer { write_batcher: w, sb: None, @@ -93,22 +85,42 @@ impl<'a> Restorer<'a> { } } - fn get_result(self) -> Result { - self.write_batcher.flush()?; + fn finalize(&mut self) -> Result<()> { + // build metadata space map + let metadata_sm_root = build_metadata_sm(self.write_batcher)?; - if self.sb.is_none() || self.discard_root.is_none() { - return Err(anyhow!("No superblock found in xml file")); - } - if self.mapping_root.is_none() || self.hint_root.is_none() { - return Err(anyhow!("No mappings or hints sections in xml file")); - } - Ok(RestoreResult { - sb: self.sb.unwrap(), - mapping_root: self.mapping_root.unwrap(), - dirty_root: self.dirty_root, - hint_root: self.hint_root.unwrap(), - discard_root: self.discard_root.unwrap(), - }) + let sb = self.sb.as_ref().unwrap(); + let mapping_root = self.mapping_root.as_ref().unwrap(); + let hint_root = self.hint_root.as_ref().unwrap(); + let discard_root = self.discard_root.as_ref().unwrap(); + let sb = Superblock { + flags: SuperblockFlags { + clean_shutdown: true, + needs_check: false, + }, + block: SUPERBLOCK_LOCATION, + version: 2, + policy_name: sb.policy.as_bytes().to_vec(), + policy_version: vec![2, 0, 0], + policy_hint_size: sb.hint_width, + metadata_sm_root, + mapping_root: *mapping_root, + dirty_root: self.dirty_root, // dirty_root is optional + hint_root: *hint_root, + discard_root: *discard_root, + discard_block_size: 0, + discard_nr_blocks: 0, + data_block_size: sb.block_size, + cache_blocks: sb.nr_cache_blocks, + compat_flags: 0, + compat_ro_flags: 0, + incompat_flags: 0, + read_hits: 0, + read_misses: 9, + write_hits: 0, + write_misses: 0, + }; + write_superblock(self.write_batcher.engine.as_ref(), SUPERBLOCK_LOCATION, &sb) } } @@ -127,6 +139,7 @@ impl<'a> MetadataVisitor for Restorer<'a> { } fn superblock_e(&mut self) -> Result { + self.finalize()?; Ok(Visit::Continue) } @@ -252,39 +265,6 @@ pub fn restore(opts: CacheRestoreOptions) -> Result<()> { // build cache mappings let mut restorer = Restorer::new(&mut w); xml::read(input, &mut restorer)?; - let result = restorer.get_result()?; - - // build metadata space map - let metadata_sm_root = build_metadata_sm(&mut w)?; - - let sb = Superblock { - flags: SuperblockFlags { - clean_shutdown: true, - needs_check: false, - }, - block: SUPERBLOCK_LOCATION, - version: 2, - policy_name: result.sb.policy.as_bytes().to_vec(), - policy_version: vec![2, 0, 0], - policy_hint_size: result.sb.hint_width, - metadata_sm_root, - mapping_root: result.mapping_root, - dirty_root: result.dirty_root, - hint_root: result.hint_root, - discard_root: result.discard_root, - discard_block_size: 0, - discard_nr_blocks: 0, - data_block_size: result.sb.block_size, - cache_blocks: result.sb.nr_cache_blocks, - compat_flags: 0, - compat_ro_flags: 0, - incompat_flags: 0, - read_hits: 0, - read_misses: 9, - write_hits: 0, - write_misses: 0, - }; - write_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?; Ok(()) } From 7239204b0149a2d3d0beedad4cf2f932d5d5ffb4 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Mon, 19 Jul 2021 11:43:03 +0800 Subject: [PATCH 7/8] [cache_repair (rust)] First pass at cache_repair --- src/bin/cache_repair.rs | 79 +++++++++++++++++++++++++++++++++++++++++ src/cache/mod.rs | 1 + src/cache/repair.rs | 68 +++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 src/bin/cache_repair.rs create mode 100644 src/cache/repair.rs diff --git a/src/bin/cache_repair.rs b/src/bin/cache_repair.rs new file mode 100644 index 0000000..d964a0f --- /dev/null +++ b/src/bin/cache_repair.rs @@ -0,0 +1,79 @@ +extern crate clap; +extern crate thinp; + +use atty::Stream; +use clap::{App, Arg}; +use std::path::Path; +use std::process; +use std::process::exit; +use std::sync::Arc; +use thinp::cache::repair::{repair, CacheRepairOptions}; +use thinp::file_utils; +use thinp::report::*; + +fn main() { + let parser = App::new("cache_repair") + .version(thinp::version::tools_version()) + .about("Repair binary cache metadata, and write it to a different device or file") + // flags + .arg( + Arg::with_name("ASYNC_IO") + .help("Force use of io_uring for synchronous io") + .long("async-io") + .hidden(true), + ) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet"), + ) + // options + .arg( + Arg::with_name("INPUT") + .help("Specify the input device") + .short("i") + .long("input") + .value_name("INPUT") + .required(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output device") + .short("o") + .long("output") + .value_name("OUTPUT") + .required(true), + ); + + let matches = parser.get_matches(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); + + if !file_utils::file_exists(input_file) { + eprintln!("Couldn't find input file '{:?}'.", &input_file); + exit(1); + } + + let report; + + if matches.is_present("QUIET") { + report = std::sync::Arc::new(mk_quiet_report()); + } else if atty::is(Stream::Stdout) { + report = std::sync::Arc::new(mk_progress_bar_report()); + } else { + report = Arc::new(mk_simple_report()); + } + + let opts = CacheRepairOptions { + input: &input_file, + output: &output_file, + async_io: matches.is_present("ASYNC_IO"), + report, + }; + + if let Err(reason) = repair(opts) { + eprintln!("{}", reason); + process::exit(1); + } +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs index c01cc6c..4a90694 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -3,6 +3,7 @@ pub mod dump; pub mod hint; pub mod ir; pub mod mapping; +pub mod repair; pub mod restore; pub mod superblock; pub mod xml; diff --git a/src/cache/repair.rs b/src/cache/repair.rs new file mode 100644 index 0000000..dd88a26 --- /dev/null +++ b/src/cache/repair.rs @@ -0,0 +1,68 @@ +use anyhow::Result; +use std::path::Path; +use std::sync::Arc; + +use crate::cache::dump::*; +use crate::cache::restore::*; +use crate::cache::superblock::*; +use crate::io_engine::*; +use crate::pdata::space_map::*; +use crate::report::*; +use crate::write_batcher::*; + +//------------------------------------------ + +pub struct CacheRepairOptions<'a> { + pub input: &'a Path, + pub output: &'a Path, + pub async_io: bool, + pub report: Arc, +} + +struct Context { + _report: Arc, + engine_in: Arc, + engine_out: Arc, +} + +const MAX_CONCURRENT_IO: u32 = 1024; + +fn new_context(opts: &CacheRepairOptions) -> Result { + let engine_in: Arc; + let engine_out: Arc; + + if opts.async_io { + engine_in = Arc::new(AsyncIoEngine::new(opts.input, MAX_CONCURRENT_IO, true)?); + engine_out = Arc::new(AsyncIoEngine::new(opts.output, MAX_CONCURRENT_IO, true)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine_in = Arc::new(SyncIoEngine::new(opts.input, nr_threads, true)?); + engine_out = Arc::new(SyncIoEngine::new(opts.output, nr_threads, true)?); + } + + Ok(Context { + _report: opts.report.clone(), + engine_in, + engine_out, + }) +} + +//------------------------------------------ + +pub fn repair(opts: CacheRepairOptions) -> Result<()> { + let ctx = new_context(&opts)?; + + let sb = read_superblock(ctx.engine_in.as_ref(), SUPERBLOCK_LOCATION)?; + + let sm = core_sm(ctx.engine_out.get_nr_blocks(), u32::MAX); + let mut w = WriteBatcher::new( + ctx.engine_out.clone(), + sm.clone(), + ctx.engine_out.get_batch_size(), + ); + let mut restorer = Restorer::new(&mut w); + + dump_metadata(ctx.engine_in, &mut restorer, &sb, true) +} + +//------------------------------------------ From 3dc01bf962b2dae5710d9320fe86344dfdb707fe Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 21 Jul 2021 21:01:05 +0800 Subject: [PATCH 8/8] [tests] Use IntoIterator on the array argument (requires Rust 1.53) --- tests/common/input_arg.rs | 5 ++--- tests/common/thin.rs | 17 ++++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/common/input_arg.rs b/tests/common/input_arg.rs index 1dc377a..3453f24 100644 --- a/tests/common/input_arg.rs +++ b/tests/common/input_arg.rs @@ -63,7 +63,7 @@ where P: InputProgram<'a>, { let args: [&str; 0] = []; - let stderr = run_fail(P::path(), &args)?; + let stderr = run_fail(P::path(), args)?; assert!(stderr.contains(P::missing_input_arg())); Ok(()) } @@ -85,8 +85,7 @@ where let mut td = TestDir::new()?; let output = mk_zeroed_md(&mut td)?; ensure_untouched(&output, || { - let args = args!["-o", &output]; - let stderr = run_fail(P::path(), &args)?; + let stderr = run_fail(P::path(), args!["-o", &output])?; assert!(stderr.contains(P::missing_input_arg())); Ok(()) }) diff --git a/tests/common/thin.rs b/tests/common/thin.rs index 685e59f..a098ae2 100644 --- a/tests/common/thin.rs +++ b/tests/common/thin.rs @@ -28,8 +28,7 @@ pub fn mk_valid_md(td: &mut TestDir) -> Result { write_xml(&xml, &mut gen)?; let _file = file_utils::create_sized_file(&md, 4096 * 4096); - let args = args!["-i", &xml, "-o", &md]; - run_ok(THIN_RESTORE, &args)?; + run_ok(THIN_RESTORE, args!["-i", &xml, "-o", &md])?; Ok(md) } @@ -40,11 +39,11 @@ pub fn mk_valid_md(td: &mut TestDir) -> Result { pub fn prep_metadata(td: &mut TestDir) -> Result { let md = mk_zeroed_md(td)?; let args = args!["-o", &md, "--format", "--nr-data-blocks", "102400"]; - run_ok(THIN_GENERATE_METADATA, &args)?; + run_ok(THIN_GENERATE_METADATA, args)?; // Create a 2GB device let args = args!["-o", &md, "--create-thin", "1"]; - run_ok(THIN_GENERATE_METADATA, &args)?; + run_ok(THIN_GENERATE_METADATA, args)?; let args = args![ "-o", &md, @@ -55,7 +54,7 @@ pub fn prep_metadata(td: &mut TestDir) -> Result { "--rw=randwrite", "--seq-nr=16" ]; - run_ok(THIN_GENERATE_MAPPINGS, &args)?; + run_ok(THIN_GENERATE_MAPPINGS, args)?; // Take a few snapshots. let mut snap_id = 2; @@ -63,7 +62,7 @@ pub fn prep_metadata(td: &mut TestDir) -> Result { // take a snapshot let snap_id_str = snap_id.to_string(); let args = args!["-o", &md, "--create-snap", &snap_id_str, "--origin", "1"]; - run_ok(THIN_GENERATE_METADATA, &args)?; + run_ok(THIN_GENERATE_METADATA, args)?; // partially overwrite the origin (64MB) let args = args![ @@ -78,7 +77,7 @@ pub fn prep_metadata(td: &mut TestDir) -> Result { "--rw=randwrite", "--seq-nr=16" ]; - run_ok(THIN_GENERATE_MAPPINGS, &args)?; + run_ok(THIN_GENERATE_MAPPINGS, args)?; snap_id += 1; } @@ -87,7 +86,7 @@ pub fn prep_metadata(td: &mut TestDir) -> Result { pub fn set_needs_check(md: &PathBuf) -> Result<()> { let args = args!["-o", &md, "--set-needs-check"]; - run_ok(THIN_GENERATE_METADATA, &args)?; + run_ok(THIN_GENERATE_METADATA, args)?; Ok(()) } @@ -111,7 +110,7 @@ pub fn generate_metadata_leaks( "--actual", &actual_str ]; - run_ok(THIN_GENERATE_DAMAGE, &args)?; + run_ok(THIN_GENERATE_DAMAGE, args)?; Ok(()) }