From 259eef9eee9d94b6b4a636ca30281eef77ae3b89 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 25 Jun 2020 10:44:57 +0100 Subject: [PATCH] [thin_shrink] calculate remaps --- Cargo.lock | 7 ++ Cargo.toml | 1 + src/bin/thin_shrink.rs | 42 +++++--- src/shrink/toplevel.rs | 212 ++++++++++++++++++++++++++++++++++++++++- src/shrink/xml.rs | 40 ++++---- 5 files changed, 265 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 508bd52..23083a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,6 +105,11 @@ dependencies = [ "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fixedbitset" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "flate2" version = "1.0.14" @@ -397,6 +402,7 @@ dependencies = [ "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)", "crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -483,6 +489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77ba37ef26c12988c1cee882d522d65e1d5d2ad8c3864665b88ee92767ed84c5" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +"checksum fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4fcacf5cd3681968f6524ea159383132937739c6c40dabab9e37ed515911b" "checksum flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" "checksum hermit-abi 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" diff --git a/Cargo.toml b/Cargo.toml index cf59ab7..dce61a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ byteorder = "1.3" clap = "2.33" crc32c = "0.4" flate2 = "1.0" +fixedbitset = "0.3" libc = "0.2.71" quick-xml = "0.18" nix = "0.17" diff --git a/src/bin/thin_shrink.rs b/src/bin/thin_shrink.rs index 7c533be..ffb29ad 100644 --- a/src/bin/thin_shrink.rs +++ b/src/bin/thin_shrink.rs @@ -7,33 +7,47 @@ use thinp::file_utils; fn main() { let parser = App::new("thin_shrink") - .version(thinp::version::TOOLS_VERSION) + .version(thinp::version::TOOLS_VERSION) .about("Rewrite xml metadata and move data in an inactive pool.") - .arg(Arg::with_name("INPUT") - .help("Specify thinp metadata xml file") - .required(true) - .long("input") - .value_name("INPUT") - .takes_value(true)) - .arg(Arg::with_name("OUTPUT") - .help("Specify output xml file") - .required(true) - .long("output") - .value_name("OUTPUT") - .takes_value(true)); + .arg( + Arg::with_name("INPUT") + .help("Specify thinp metadata xml file") + .required(true) + .long("input") + .value_name("INPUT") + .takes_value(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify output xml file") + .required(true) + .long("output") + .value_name("OUTPUT") + .takes_value(true), + ) + // FIXME: support various disk units + .arg( + Arg::with_name("SIZE") + .help("Specify new size for the pool (in data blocks)") + .required(true) + .long("nr-blocks") + .value_name("SIZE") + .takes_value(true), + ); let matches = parser.get_matches(); // FIXME: check these look like xml let input_file = matches.value_of("INPUT").unwrap(); let output_file = matches.value_of("OUTPUT").unwrap(); + let size = matches.value_of("SIZE").unwrap().parse::().unwrap(); if !file_utils::file_exists(input_file) { eprintln!("Couldn't find input file '{}'.", &input_file); exit(1); } - if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file) { + if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, size) { println!("Application error: {}\n", reason); exit(1); } diff --git a/src/shrink/toplevel.rs b/src/shrink/toplevel.rs index 53dfbe1..0e9da5b 100644 --- a/src/shrink/toplevel.rs +++ b/src/shrink/toplevel.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use fixedbitset::{FixedBitSet, IndexRange}; use std::fs::OpenOptions; use std::os::unix::fs::OpenOptionsExt; @@ -6,7 +7,178 @@ use crate::shrink::xml; //--------------------------------------- -pub fn shrink(input_file: &str, _output_file: &str) -> Result<()> { +#[derive(Debug)] +struct Pass1 { + // FIXME: Inefficient, use a range_set of some description + allocated_blocks: FixedBitSet, + + nr_blocks: u64, + + /// High blocks are beyond the new, reduced end of the pool. These + /// will need to be moved. + nr_high_blocks: u64, +} + +impl Pass1 { + fn new(nr_blocks: u64) -> Pass1 { + Pass1 { + allocated_blocks: FixedBitSet::with_capacity(0), + nr_blocks, + nr_high_blocks: 0, + } + } +} + +impl xml::MetadataVisitor for Pass1 { + fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<()> { + self.allocated_blocks.grow(sb.nr_data_blocks as usize); + Ok(()) + } + + fn superblock_e(&mut self) -> Result<()> { + Ok(()) + } + + fn device_b(&mut self, _d: &xml::Device) -> Result<()> { + Ok(()) + } + + fn device_e(&mut self) -> Result<()> { + Ok(()) + } + + fn map(&mut self, m: xml::Map) -> Result<()> { + for i in m.data_begin..(m.data_begin + m.len) { + if i > self.nr_blocks { + self.nr_high_blocks += 1; + } + self.allocated_blocks.insert(i as usize); + } + Ok(()) + } + + fn eof(&mut self) -> Result<()> { + Ok(()) + } +} + +type BlockRange = std::ops::Range; + +fn bits_to_ranges(bits: &FixedBitSet) -> Vec { + let mut ranges = Vec::new(); + let mut start = None; + + for i in 0..bits.len() { + match (bits[i], start) { + (false, None) => {} + (true, None) => { + start = Some((i as u64, 1)); + } + (false, Some((b, len))) => { + ranges.push(b..(b + len)); + start = None; + } + (true, Some((b, len))) => { + start = Some((b, len + 1)); + } + } + } + + if let Some((b, len)) = start { + ranges.push(b..(b + len)); + } + + ranges +} + +// Splits the ranges into those below threshold, and those equal or +// above threshold below threshold, and those equal or above threshold +fn ranges_split(ranges: &Vec, threshold: u64) -> (Vec, Vec) { + use std::ops::Range; + + let mut below = Vec::new(); + let mut above = Vec::new(); + for r in ranges { + match r { + Range { start, end } if *end <= threshold => below.push(*start..*end), + Range { start, end } if *start < threshold => { + below.push(*start..threshold); + above.push(threshold..*end); + } + Range { start, end } => above.push(*start..*end), + } + } + (below, above) +} + +fn negate_ranges(ranges: &Vec) -> Vec { + use std::ops::Range; + + let mut result = Vec::new(); + let mut cursor = 0; + + for r in ranges { + match r { + Range { start, end } if cursor < *start => { + result.push(cursor..*start); + cursor = *end; + } + Range { start: _, end } => { + cursor = *end; + } + } + } + + result +} + +fn range_len(r: &BlockRange) -> u64 { + r.end - r.start +} + +fn ranges_total(rs: &Vec) -> u64 { + rs.into_iter().fold(0, |sum, r| sum + range_len(r)) +} + +// Assumes there is enough space to remap. +fn remap_ranges(ranges: Vec, free: Vec) -> Vec<(BlockRange, BlockRange)> { + use std::cmp::Ordering; + + let mut remap = Vec::new(); + let mut range_iter = ranges.into_iter(); + let mut free_iter = free.into_iter(); + + let mut r_ = range_iter.next(); + let mut f_ = free_iter.next(); + + while let (Some(r), Some(f)) = (r_, f_) { + let rlen = range_len(&r); + let flen = range_len(&f); + + match rlen.cmp(&flen) { + Ordering::Less => { + // range fits into the free chunk + remap.push((r, f.start..(f.start + rlen))); + f_ = Some((f.start + rlen)..f.end); + r_ = range_iter.next(); + }, + Ordering::Equal => { + remap.push((r, f)); + r_ = range_iter.next(); + f_ = free_iter.next(); + }, + Ordering::Greater => { + remap.push((r.start..(r.start + flen), f)); + r_ = Some((r.start + flen)..r.end); + f_ = free_iter.next(); + } + } + } + + remap +} + +pub fn shrink(input_file: &str, _output_file: &str, nr_blocks: u64) -> Result<()> { let input = OpenOptions::new() .read(true) .write(false) @@ -14,8 +186,42 @@ pub fn shrink(input_file: &str, _output_file: &str) -> Result<()> { .open(input_file)?; // let mut visitor = xml::XmlWriter::new(std::io::stdout()); - let mut visitor = xml::NoopVisitor::new(); - xml::read(input, &mut visitor)?; + // let mut visitor = xml::NoopVisitor::new(); + let mut pass1 = Pass1::new(nr_blocks); + xml::read(input, &mut pass1)?; + eprintln!("{} blocks need moving", pass1.nr_high_blocks); + + let mut free_blocks = 0u64; + for i in 0..pass1.allocated_blocks.len() { + if !pass1.allocated_blocks[i] { + free_blocks += 1; + } + } + eprintln!("{} free blocks below new end.", free_blocks); + + let ranges = bits_to_ranges(&pass1.allocated_blocks); + eprintln!("{} allocated ranges:", ranges.len()); + + eprintln!("{:?}", &ranges); + + let (below, above) = ranges_split(&ranges, nr_blocks); + eprintln!("ranges split at {}: ({:?}, {:?})", nr_blocks, below, above); + + let free = negate_ranges(&below); + eprintln!("free {:?}.", free); + + let nr_moving = ranges_total(&above); + eprintln!("{} blocks need to be remapped.", nr_moving); + + let free_blocks = ranges_total(&free); + eprintln!("{} free blocks.", free_blocks); + + if free_blocks < nr_moving { + panic!("Insufficient space"); + } + + let remaps = remap_ranges(above, free); + eprintln!("remappings {:?}.", remaps); Ok(()) } diff --git a/src/shrink/xml.rs b/src/shrink/xml.rs index f060016..1a3c9b7 100644 --- a/src/shrink/xml.rs +++ b/src/shrink/xml.rs @@ -14,29 +14,29 @@ use quick_xml::{Reader, Writer}; //--------------------------------------- pub struct Superblock { - uuid: String, - time: u64, - transaction: u64, - flags: Option, - version: Option, - data_block_size: u32, - nr_data_blocks: u64, - metadata_snap: Option, + pub uuid: String, + pub time: u64, + pub transaction: u64, + pub flags: Option, + pub version: Option, + pub data_block_size: u32, + pub nr_data_blocks: u64, + pub metadata_snap: Option, } pub struct Device { - dev_id: u32, - mapped_blocks: u64, - transaction: u64, - creation_time: u64, - snap_time: u64, + pub dev_id: u32, + pub mapped_blocks: u64, + pub transaction: u64, + pub creation_time: u64, + pub snap_time: u64, } pub struct Map { - thin_begin: u64, - data_begin: u64, - time: u32, - len: u64, + pub thin_begin: u64, + pub data_begin: u64, + pub time: u32, + pub len: u64, } pub trait MetadataVisitor { @@ -238,11 +238,11 @@ fn parse_superblock(e: &BytesStart) -> Result { uuid: check_attr(tag, "uuid", uuid)?, time: check_attr(tag, "time", time)?, transaction: check_attr(tag, "transaction", transaction)?, - flags: flags, - version: version, + flags, + version, data_block_size: check_attr(tag, "data_block_size", data_block_size)?, nr_data_blocks: check_attr(tag, "nr_data_blocks", nr_data_blocks)?, - metadata_snap: metadata_snap, + metadata_snap, }) }