[thin_shrink] calculate remaps

This commit is contained in:
Joe Thornber 2020-06-25 10:44:57 +01:00
parent 3f1b776359
commit 259eef9eee
5 changed files with 265 additions and 37 deletions

7
Cargo.lock generated
View File

@ -105,6 +105,11 @@ dependencies = [
"regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "fixedbitset"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "flate2" name = "flate2"
version = "1.0.14" version = "1.0.14"
@ -397,6 +402,7 @@ dependencies = [
"byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)",
"crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)",
"nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -483,6 +489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77ba37ef26c12988c1cee882d522d65e1d5d2ad8c3864665b88ee92767ed84c5" "checksum crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77ba37ef26c12988c1cee882d522d65e1d5d2ad8c3864665b88ee92767ed84c5"
"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
"checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
"checksum fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4fcacf5cd3681968f6524ea159383132937739c6c40dabab9e37ed515911b"
"checksum flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" "checksum flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42"
"checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
"checksum hermit-abi 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" "checksum hermit-abi 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71"

View File

@ -11,6 +11,7 @@ byteorder = "1.3"
clap = "2.33" clap = "2.33"
crc32c = "0.4" crc32c = "0.4"
flate2 = "1.0" flate2 = "1.0"
fixedbitset = "0.3"
libc = "0.2.71" libc = "0.2.71"
quick-xml = "0.18" quick-xml = "0.18"
nix = "0.17" nix = "0.17"

View File

@ -7,33 +7,47 @@ use thinp::file_utils;
fn main() { fn main() {
let parser = App::new("thin_shrink") let parser = App::new("thin_shrink")
.version(thinp::version::TOOLS_VERSION) .version(thinp::version::TOOLS_VERSION)
.about("Rewrite xml metadata and move data in an inactive pool.") .about("Rewrite xml metadata and move data in an inactive pool.")
.arg(Arg::with_name("INPUT") .arg(
.help("Specify thinp metadata xml file") Arg::with_name("INPUT")
.required(true) .help("Specify thinp metadata xml file")
.long("input") .required(true)
.value_name("INPUT") .long("input")
.takes_value(true)) .value_name("INPUT")
.arg(Arg::with_name("OUTPUT") .takes_value(true),
.help("Specify output xml file") )
.required(true) .arg(
.long("output") Arg::with_name("OUTPUT")
.value_name("OUTPUT") .help("Specify output xml file")
.takes_value(true)); .required(true)
.long("output")
.value_name("OUTPUT")
.takes_value(true),
)
// FIXME: support various disk units
.arg(
Arg::with_name("SIZE")
.help("Specify new size for the pool (in data blocks)")
.required(true)
.long("nr-blocks")
.value_name("SIZE")
.takes_value(true),
);
let matches = parser.get_matches(); let matches = parser.get_matches();
// FIXME: check these look like xml // FIXME: check these look like xml
let input_file = matches.value_of("INPUT").unwrap(); let input_file = matches.value_of("INPUT").unwrap();
let output_file = matches.value_of("OUTPUT").unwrap(); let output_file = matches.value_of("OUTPUT").unwrap();
let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap();
if !file_utils::file_exists(input_file) { if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file); eprintln!("Couldn't find input file '{}'.", &input_file);
exit(1); exit(1);
} }
if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file) { if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, size) {
println!("Application error: {}\n", reason); println!("Application error: {}\n", reason);
exit(1); exit(1);
} }

View File

@ -1,4 +1,5 @@
use anyhow::Result; use anyhow::Result;
use fixedbitset::{FixedBitSet, IndexRange};
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::os::unix::fs::OpenOptionsExt; use std::os::unix::fs::OpenOptionsExt;
@ -6,7 +7,178 @@ use crate::shrink::xml;
//--------------------------------------- //---------------------------------------
pub fn shrink(input_file: &str, _output_file: &str) -> Result<()> { #[derive(Debug)]
struct Pass1 {
// FIXME: Inefficient, use a range_set of some description
allocated_blocks: FixedBitSet,
nr_blocks: u64,
/// High blocks are beyond the new, reduced end of the pool. These
/// will need to be moved.
nr_high_blocks: u64,
}
impl Pass1 {
fn new(nr_blocks: u64) -> Pass1 {
Pass1 {
allocated_blocks: FixedBitSet::with_capacity(0),
nr_blocks,
nr_high_blocks: 0,
}
}
}
impl xml::MetadataVisitor for Pass1 {
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<()> {
self.allocated_blocks.grow(sb.nr_data_blocks as usize);
Ok(())
}
fn superblock_e(&mut self) -> Result<()> {
Ok(())
}
fn device_b(&mut self, _d: &xml::Device) -> Result<()> {
Ok(())
}
fn device_e(&mut self) -> Result<()> {
Ok(())
}
fn map(&mut self, m: xml::Map) -> Result<()> {
for i in m.data_begin..(m.data_begin + m.len) {
if i > self.nr_blocks {
self.nr_high_blocks += 1;
}
self.allocated_blocks.insert(i as usize);
}
Ok(())
}
fn eof(&mut self) -> Result<()> {
Ok(())
}
}
type BlockRange = std::ops::Range<u64>;
fn bits_to_ranges(bits: &FixedBitSet) -> Vec<BlockRange> {
let mut ranges = Vec::new();
let mut start = None;
for i in 0..bits.len() {
match (bits[i], start) {
(false, None) => {}
(true, None) => {
start = Some((i as u64, 1));
}
(false, Some((b, len))) => {
ranges.push(b..(b + len));
start = None;
}
(true, Some((b, len))) => {
start = Some((b, len + 1));
}
}
}
if let Some((b, len)) = start {
ranges.push(b..(b + len));
}
ranges
}
// Splits the ranges into those below threshold, and those equal or
// above threshold below threshold, and those equal or above threshold
fn ranges_split(ranges: &Vec<BlockRange>, threshold: u64) -> (Vec<BlockRange>, Vec<BlockRange>) {
use std::ops::Range;
let mut below = Vec::new();
let mut above = Vec::new();
for r in ranges {
match r {
Range { start, end } if *end <= threshold => below.push(*start..*end),
Range { start, end } if *start < threshold => {
below.push(*start..threshold);
above.push(threshold..*end);
}
Range { start, end } => above.push(*start..*end),
}
}
(below, above)
}
fn negate_ranges(ranges: &Vec<BlockRange>) -> Vec<BlockRange> {
use std::ops::Range;
let mut result = Vec::new();
let mut cursor = 0;
for r in ranges {
match r {
Range { start, end } if cursor < *start => {
result.push(cursor..*start);
cursor = *end;
}
Range { start: _, end } => {
cursor = *end;
}
}
}
result
}
fn range_len(r: &BlockRange) -> u64 {
r.end - r.start
}
fn ranges_total(rs: &Vec<BlockRange>) -> u64 {
rs.into_iter().fold(0, |sum, r| sum + range_len(r))
}
// Assumes there is enough space to remap.
fn remap_ranges(ranges: Vec<BlockRange>, free: Vec<BlockRange>) -> Vec<(BlockRange, BlockRange)> {
use std::cmp::Ordering;
let mut remap = Vec::new();
let mut range_iter = ranges.into_iter();
let mut free_iter = free.into_iter();
let mut r_ = range_iter.next();
let mut f_ = free_iter.next();
while let (Some(r), Some(f)) = (r_, f_) {
let rlen = range_len(&r);
let flen = range_len(&f);
match rlen.cmp(&flen) {
Ordering::Less => {
// range fits into the free chunk
remap.push((r, f.start..(f.start + rlen)));
f_ = Some((f.start + rlen)..f.end);
r_ = range_iter.next();
},
Ordering::Equal => {
remap.push((r, f));
r_ = range_iter.next();
f_ = free_iter.next();
},
Ordering::Greater => {
remap.push((r.start..(r.start + flen), f));
r_ = Some((r.start + flen)..r.end);
f_ = free_iter.next();
}
}
}
remap
}
pub fn shrink(input_file: &str, _output_file: &str, nr_blocks: u64) -> Result<()> {
let input = OpenOptions::new() let input = OpenOptions::new()
.read(true) .read(true)
.write(false) .write(false)
@ -14,8 +186,42 @@ pub fn shrink(input_file: &str, _output_file: &str) -> Result<()> {
.open(input_file)?; .open(input_file)?;
// let mut visitor = xml::XmlWriter::new(std::io::stdout()); // let mut visitor = xml::XmlWriter::new(std::io::stdout());
let mut visitor = xml::NoopVisitor::new(); // let mut visitor = xml::NoopVisitor::new();
xml::read(input, &mut visitor)?; let mut pass1 = Pass1::new(nr_blocks);
xml::read(input, &mut pass1)?;
eprintln!("{} blocks need moving", pass1.nr_high_blocks);
let mut free_blocks = 0u64;
for i in 0..pass1.allocated_blocks.len() {
if !pass1.allocated_blocks[i] {
free_blocks += 1;
}
}
eprintln!("{} free blocks below new end.", free_blocks);
let ranges = bits_to_ranges(&pass1.allocated_blocks);
eprintln!("{} allocated ranges:", ranges.len());
eprintln!("{:?}", &ranges);
let (below, above) = ranges_split(&ranges, nr_blocks);
eprintln!("ranges split at {}: ({:?}, {:?})", nr_blocks, below, above);
let free = negate_ranges(&below);
eprintln!("free {:?}.", free);
let nr_moving = ranges_total(&above);
eprintln!("{} blocks need to be remapped.", nr_moving);
let free_blocks = ranges_total(&free);
eprintln!("{} free blocks.", free_blocks);
if free_blocks < nr_moving {
panic!("Insufficient space");
}
let remaps = remap_ranges(above, free);
eprintln!("remappings {:?}.", remaps);
Ok(()) Ok(())
} }

View File

@ -14,29 +14,29 @@ use quick_xml::{Reader, Writer};
//--------------------------------------- //---------------------------------------
pub struct Superblock { pub struct Superblock {
uuid: String, pub uuid: String,
time: u64, pub time: u64,
transaction: u64, pub transaction: u64,
flags: Option<u32>, pub flags: Option<u32>,
version: Option<u32>, pub version: Option<u32>,
data_block_size: u32, pub data_block_size: u32,
nr_data_blocks: u64, pub nr_data_blocks: u64,
metadata_snap: Option<u64>, pub metadata_snap: Option<u64>,
} }
pub struct Device { pub struct Device {
dev_id: u32, pub dev_id: u32,
mapped_blocks: u64, pub mapped_blocks: u64,
transaction: u64, pub transaction: u64,
creation_time: u64, pub creation_time: u64,
snap_time: u64, pub snap_time: u64,
} }
pub struct Map { pub struct Map {
thin_begin: u64, pub thin_begin: u64,
data_begin: u64, pub data_begin: u64,
time: u32, pub time: u32,
len: u64, pub len: u64,
} }
pub trait MetadataVisitor { pub trait MetadataVisitor {
@ -238,11 +238,11 @@ fn parse_superblock(e: &BytesStart) -> Result<Superblock> {
uuid: check_attr(tag, "uuid", uuid)?, uuid: check_attr(tag, "uuid", uuid)?,
time: check_attr(tag, "time", time)?, time: check_attr(tag, "time", time)?,
transaction: check_attr(tag, "transaction", transaction)?, transaction: check_attr(tag, "transaction", transaction)?,
flags: flags, flags,
version: version, version,
data_block_size: check_attr(tag, "data_block_size", data_block_size)?, data_block_size: check_attr(tag, "data_block_size", data_block_size)?,
nr_data_blocks: check_attr(tag, "nr_data_blocks", nr_data_blocks)?, nr_data_blocks: check_attr(tag, "nr_data_blocks", nr_data_blocks)?,
metadata_snap: metadata_snap, metadata_snap,
}) })
} }