emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into 2020-06-13-thin-check-rewrite

Browse Source
This commit is contained in:
Joe Thornber
2020-07-27 15:53:26 +01:00
parent 5e19029e65 e9055f5d7d
commit 3cf6307762
33 changed files with 2610 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/bin/thin_check.cc → src/bin/thin_check.rs
View File

11
src/bin/thin_metadata_pack.rs
View File

@@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use std::process::exit;
use thinp::file_utils;
@@ -23,14 +24,14 @@ fn main() {
.takes_value(true));
let matches = parser.get_matches();
let input_file = matches.value_of("INPUT").unwrap();
let output_file = matches.value_of("OUTPUT").unwrap();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file);
if !file_utils::file_exists(&input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file.display());
exit(1);
}
if let Err(reason) = thinp::pack::toplevel::pack(&input_file, &output_file) {
println!("Application error: {}\n", reason);
exit(1);

39
src/bin/thin_metadata_unpack.rs
View File

@@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use std::process;
use thinp::file_utils;
@@ -9,30 +10,34 @@ use std::process::exit;
fn main() {
let parser = App::new("thin_metadata_unpack")
.version(thinp::version::TOOLS_VERSION)
.version(thinp::version::TOOLS_VERSION)
.about("Unpack a compressed file of thin metadata.")
.arg(Arg::with_name("INPUT")
.help("Specify thinp metadata binary device/file")
.required(true)
.short("i")
.value_name("DEV")
.takes_value(true))
.arg(Arg::with_name("OUTPUT")
.help("Specify packed output file")
.required(true)
.short("o")
.value_name("FILE")
.takes_value(true));
.arg(
Arg::with_name("INPUT")
.help("Specify thinp metadata binary device/file")
.required(true)
.short("i")
.value_name("DEV")
.takes_value(true),
)
.arg(
Arg::with_name("OUTPUT")
.help("Specify packed output file")
.required(true)
.short("o")
.value_name("FILE")
.takes_value(true),
);
let matches = parser.get_matches();
let input_file = matches.value_of("INPUT").unwrap();
let output_file = matches.value_of("OUTPUT").unwrap();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file);
eprintln!("Couldn't find input file '{}'.", &input_file.display());
exit(1);
}
if let Err(reason) = thinp::pack::toplevel::unpack(&input_file, &output_file) {
println!("Application error: {}", reason);
process::exit(1);

74
src/bin/thin_shrink.rs Normal file
View File

@@ -0,0 +1,74 @@
extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use std::process::exit;
use thinp::file_utils;
fn main() {
let parser = App::new("thin_shrink")
.version(thinp::version::TOOLS_VERSION)
.about("Rewrite xml metadata and move data in an inactive pool.")
.arg(
Arg::with_name("INPUT")
.help("Specify thinp metadata xml file")
.required(true)
.long("input")
.value_name("INPUT")
.takes_value(true),
)
.arg(
Arg::with_name("OUTPUT")
.help("Specify output xml file")
.required(true)
.long("output")
.value_name("OUTPUT")
.takes_value(true),
)
.arg(
Arg::with_name("DATA")
.help("Specify pool data device where data will be moved")
.required(true)
.long("data")
.value_name("DATA")
.takes_value(true),
)
.arg(
Arg::with_name("NOCOPY")
.help("Skip the copying of data, useful for benchmarking")
.required(false)
.long("no-copy")
.value_name("NOCOPY")
.takes_value(false),
)
.arg(
Arg::with_name("SIZE")
.help("Specify new size for the pool (in data blocks)")
.required(true)
.long("nr-blocks")
.value_name("SIZE")
.takes_value(true),
);
let matches = parser.get_matches();
// FIXME: check these look like xml
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap();
let data_file = Path::new(matches.value_of("DATA").unwrap());
let do_copy = !matches.is_present("NOCOPY");
if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", input_file.display());
exit(1);
}
if let Err(reason) =
thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy)
{
println!("Application error: {}\n", reason);
exit(1);
}
}

67
src/file_utils.rs
View File

@@ -1,8 +1,11 @@
use nix::sys::stat;
use nix::sys::stat::{FileStat, SFlag};
use std::fs::{File, OpenOptions};
use std::io;
use std::fs::File;
use std::io::{Seek, Write};
use std::os::unix::io::AsRawFd;
use std::path::Path;
use tempfile::tempfile;
//---------------------------------------
@@ -11,15 +14,13 @@ fn check_bits(mode: u32, flag: &SFlag) -> bool {
}
pub fn is_file_or_blk(info: FileStat) -> bool {
check_bits(info.st_mode, &stat::SFlag::S_IFBLK) ||
check_bits(info.st_mode, &stat::SFlag::S_IFREG)
check_bits(info.st_mode, &stat::SFlag::S_IFBLK)
|| check_bits(info.st_mode, &stat::SFlag::S_IFREG)
}
pub fn file_exists(path: &str) -> bool {
pub fn file_exists(path: &Path) -> bool {
match stat::stat(path) {
Ok(info) => {
is_file_or_blk(info)
}
Ok(info) => is_file_or_blk(info),
_ => {
// FIXME: assuming all errors indicate the file doesn't
// exist.
@@ -39,19 +40,19 @@ pub fn fail<T>(msg: &str) -> io::Result<T> {
Err(e)
}
fn get_device_size(path: &str) -> io::Result<u64> {
let file = File::open(path)?;
fn get_device_size(path: &Path) -> io::Result<u64> {
let file = File::open(path)?;
let fd = file.as_raw_fd();
let mut cap = 0u64;
unsafe {
match ioctl_blkgetsize64(fd, &mut cap) {
Ok(_) => {Ok(cap)}
_ => {fail("BLKGETSIZE64 ioctl failed")}
}
match ioctl_blkgetsize64(fd, &mut cap) {
Ok(_) => Ok(cap),
_ => fail("BLKGETSIZE64 ioctl failed"),
}
}
}
pub fn file_size(path: &str) -> io::Result<u64> {
pub fn file_size(path: &Path) -> io::Result<u64> {
match stat::stat(path) {
Ok(info) => {
if check_bits(info.st_mode, &SFlag::S_IFREG) {
@@ -60,12 +61,40 @@ pub fn file_size(path: &str) -> io::Result<u64> {
get_device_size(path)
} else {
fail("not a regular file or block device")
}
}
}
_ => {
fail("stat failed")
}
}
_ => fail("stat failed"),
}
}
//---------------------------------------
fn set_size<W: Write + Seek>(w: &mut W, nr_bytes: u64) -> io::Result<()> {
let zeroes: Vec<u8> = vec![0; 1];
if nr_bytes > 0 {
w.seek(io::SeekFrom::Start(nr_bytes - 1))?;
w.write_all(&zeroes)?;
}
Ok(())
}
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = tempfile()?;
set_size(&mut file, nr_bytes)?;
Ok(file)
}
pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
set_size(&mut file, nr_bytes)?;
Ok(file)
}
//---------------------------------------

3
src/lib.rs
View File

@@ -1,3 +1,4 @@
extern crate anyhow;
extern crate byteorder;
extern crate crc32c;
extern crate flate2;
@@ -18,6 +19,8 @@ pub mod block_manager;
pub mod check;
pub mod file_utils;
pub mod pack;
pub mod shrink;
pub mod thin;
pub mod version;
pub mod thin;
pub mod checksum;

33
src/pack/toplevel.rs
View File

@@ -11,6 +11,7 @@ use std::{
io::prelude::*,
io::Write,
ops::DerefMut,
path::Path,
sync::{Arc, Mutex},
thread::spawn,
};
@@ -18,9 +19,9 @@ use std::{
use rand::prelude::*;
use std::sync::mpsc::{sync_channel, Receiver};
use crate::checksum::*;
use crate::file_utils;
use crate::pack::node_encode::*;
use crate::checksum::*;
const BLOCK_SIZE: u64 = 4096;
const MAGIC: u64 = 0xa537a0aa6309ef77;
@@ -31,11 +32,6 @@ fn shuffle<T>(v: &mut Vec<T>) {
v.shuffle(&mut rng);
}
// FIXME: move to a utils module
fn div_up(n: u64, d: u64) -> u64 {
(n + d - 1) / d
}
// Each thread processes multiple contiguous runs of blocks, called
// chunks. Chunks are shuffled so each thread gets chunks spread
// across the dev in case there are large regions that don't contain
@@ -44,12 +40,17 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec<Vec<(u64, u64)>> {
use std::cmp::{max, min};
let chunk_size = min(4 * 1024u64, max(128u64, nr_blocks / (nr_jobs * 64)));
let nr_chunks = div_up(nr_blocks, chunk_size);
let nr_chunks = nr_blocks / chunk_size;
let mut chunks = Vec::with_capacity(nr_chunks as usize);
for i in 0..nr_chunks {
chunks.push((i * chunk_size, (i + 1) * chunk_size));
}
// there may be a smaller chunk at the back of the file.
if nr_chunks * chunk_size < nr_blocks {
chunks.push((nr_chunks * chunk_size, nr_blocks));
}
shuffle(&mut chunks);
let mut vs = Vec::with_capacity(nr_jobs as usize);
@@ -64,8 +65,8 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec<Vec<(u64, u64)>> {
vs
}
pub fn pack(input_file: &str, output_file: &str) -> Result<()> {
let nr_blocks = get_nr_blocks(&input_file).context("getting nr blocks")?;
pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
let nr_blocks = get_nr_blocks(&input_file)?;
let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128));
let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs);
@@ -101,11 +102,7 @@ pub fn pack(input_file: &str, output_file: &str) -> Result<()> {
Ok(())
}
fn crunch<R, W>(
input: Arc<Mutex<R>>,
output: Arc<Mutex<W>>,
ranges: Vec<(u64, u64)>,
) -> Result<()>
fn crunch<R, W>(input: Arc<Mutex<R>>, output: Arc<Mutex<W>>, ranges: Vec<(u64, u64)>) -> Result<()>
where
R: Read + Seek,
W: Write,
@@ -189,7 +186,7 @@ where
r.read_u64::<LittleEndian>()
}
fn get_nr_blocks(path: &str) -> io::Result<u64> {
fn get_nr_blocks(path: &Path) -> io::Result<u64> {
let len = file_utils::file_size(path)?;
Ok(len / (BLOCK_SIZE as u64))
}
@@ -212,9 +209,7 @@ fn pack_block<W: Write>(w: &mut W, kind: BT, buf: &[u8]) -> Result<()> {
BT::NODE => pack_btree_node(w, buf).context("unable to pack btree node")?,
BT::INDEX => pack_index(w, buf).context("unable to pack space map index")?,
BT::BITMAP => pack_bitmap(w, buf).context("unable to pack space map bitmap")?,
BT::UNKNOWN => {
return Err(anyhow!("asked to pack an unknown block type"))
}
BT::UNKNOWN => return Err(anyhow!("asked to pack an unknown block type")),
}
Ok(())
@@ -266,7 +261,7 @@ where
Ok(())
}
pub fn unpack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> {
pub fn unpack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
let mut input = OpenOptions::new()
.read(true)
.write(false)

61
src/shrink/copier.rs Normal file
View File

@@ -0,0 +1,61 @@
use anyhow::Result;
use std::fs::OpenOptions;
use std::path::Path;
use std::io::{Seek, SeekFrom, Write, Read};
//use std::os::unix::fs::OpenOptionsExt;
pub type Sector = u64;
#[derive(Debug)]
pub struct Region {
pub src: Sector,
pub dest: Sector,
pub len: Sector,
}
fn copy_step<W>(file: &mut W, src_byte: u64, dest_byte: u64, len: usize) -> Result<()>
where
W: Write + Seek + Read,
{
let mut buf = vec![0; len];
file.seek(SeekFrom::Start(src_byte))?;
file.read_exact(&mut buf)?;
file.seek(SeekFrom::Start(dest_byte))?;
file.write_all(&buf)?;
Ok(())
}
fn copy_region<W>(file: &mut W, r: &Region) -> Result<()>
where
W: Write + Seek + Read,
{
const MAX_BYTES: Sector = 1024 * 1024 * 64;
let src_bytes = r.src * 512;
let dest_bytes = r.dest * 512;
let len_bytes = r.len * 512;
let mut written = 0;
while written != len_bytes {
let step = u64::min(len_bytes - written, MAX_BYTES);
copy_step(file, src_bytes + written, dest_bytes + written, step as usize)?;
written += step;
}
Ok(())
}
pub fn copy(path: &Path, regions: &[Region]) -> Result<()> {
let mut input = OpenOptions::new()
.read(true)
.write(true)
//.custom_flags(libc::O_DIRECT)
.open(path)?;
for r in regions {
eprintln!("copying {:?}", r);
copy_region(&mut input, r)?;
}
input.flush()?;
Ok(())
}

3
src/shrink/mod.rs Normal file
View File

@@ -0,0 +1,3 @@
pub mod toplevel;
mod copier;

528
src/shrink/toplevel.rs Normal file
View File

@@ -0,0 +1,528 @@
use anyhow::{anyhow, Result};
use fixedbitset::FixedBitSet;
use std::fs::OpenOptions;
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use crate::shrink::copier::{self, Region};
use crate::thin::xml::{self, Visit};
//---------------------------------------
#[derive(Debug)]
struct Pass1 {
// FIXME: Inefficient, use a range_set of some description
allocated_blocks: FixedBitSet,
nr_blocks: u64,
/// High blocks are beyond the new, reduced end of the pool. These
/// will need to be moved.
nr_high_blocks: u64,
block_size: Option<u64>,
}
impl Pass1 {
fn new(nr_blocks: u64) -> Pass1 {
Pass1 {
allocated_blocks: FixedBitSet::with_capacity(0),
nr_blocks,
nr_high_blocks: 0,
block_size: None,
}
}
}
impl xml::MetadataVisitor for Pass1 {
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
self.allocated_blocks.grow(sb.nr_data_blocks as usize);
self.block_size = Some(sb.data_block_size as u64);
Ok(Visit::Continue)
}
fn superblock_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_b(&mut self, _d: &xml::Device) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn map(&mut self, m: &xml::Map) -> Result<Visit> {
for i in m.data_begin..(m.data_begin + m.len) {
if i > self.nr_blocks {
self.nr_high_blocks += 1;
}
self.allocated_blocks.insert(i as usize);
}
Ok(Visit::Continue)
}
fn eof(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
}
//---------------------------------------
// Writes remapped xml
struct Pass2<W: Write> {
writer: xml::XmlWriter<W>,
nr_blocks: u64,
remaps: Vec<(BlockRange, BlockRange)>,
}
impl<W: Write> Pass2<W> {
fn new(w: W, nr_blocks: u64, remaps: Vec<(BlockRange, BlockRange)>) -> Pass2<W> {
Pass2 {
writer: xml::XmlWriter::new(w),
nr_blocks,
remaps,
}
}
}
impl<W: Write> xml::MetadataVisitor for Pass2<W> {
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
self.writer.superblock_b(sb)
}
fn superblock_e(&mut self) -> Result<Visit> {
self.writer.superblock_e()
}
fn device_b(&mut self, d: &xml::Device) -> Result<Visit> {
self.writer.device_b(d)
}
fn device_e(&mut self) -> Result<Visit> {
self.writer.device_e()
}
fn map(&mut self, m: &xml::Map) -> Result<Visit> {
if m.data_begin + m.len < self.nr_blocks {
// no remapping needed.
self.writer.map(m)?;
} else {
let r = m.data_begin..(m.data_begin + m.len);
let remaps = remap(&r, &self.remaps);
let mut written = 0;
for r in remaps {
self.writer.map(&xml::Map {
thin_begin: m.thin_begin + written,
data_begin: r.start,
time: m.time,
len: range_len(&r),
})?;
written += range_len(&r);
}
}
Ok(Visit::Continue)
}
fn eof(&mut self) -> Result<Visit> {
self.writer.eof()
}
}
//---------------------------------------
type BlockRange = std::ops::Range<u64>;
fn bits_to_ranges(bits: &FixedBitSet) -> Vec<BlockRange> {
let mut ranges = Vec::new();
let mut start = None;
for i in 0..bits.len() {
match (bits[i], start) {
(false, None) => {}
(true, None) => {
start = Some((i as u64, 1));
}
(false, Some((b, len))) => {
ranges.push(b..(b + len));
start = None;
}
(true, Some((b, len))) => {
start = Some((b, len + 1));
}
}
}
if let Some((b, len)) = start {
ranges.push(b..(b + len));
}
ranges
}
// Splits the ranges into those below threshold, and those equal or
// above threshold below threshold, and those equal or above threshold
fn ranges_split(ranges: &[BlockRange], threshold: u64) -> (Vec<BlockRange>, Vec<BlockRange>) {
use std::ops::Range;
let mut below = Vec::new();
let mut above = Vec::new();
for r in ranges {
match r {
Range { start, end } if *end <= threshold => below.push(*start..*end),
Range { start, end } if *start < threshold => {
below.push(*start..threshold);
above.push(threshold..*end);
}
Range { start, end } => above.push(*start..*end),
}
}
(below, above)
}
fn negate_ranges(ranges: &[BlockRange], upper_limit: u64) -> Vec<BlockRange> {
use std::ops::Range;
let mut result = Vec::new();
let mut cursor = 0;
for r in ranges {
match r {
Range { start, end } if cursor < *start => {
result.push(cursor..*start);
cursor = *end;
}
Range { start: _, end } => {
cursor = *end;
}
}
}
if cursor < upper_limit {
result.push(cursor..upper_limit);
}
result
}
fn range_len(r: &BlockRange) -> u64 {
r.end - r.start
}
fn ranges_total(rs: &[BlockRange]) -> u64 {
rs.iter().fold(0, |sum, r| sum + range_len(r))
}
// Assumes there is enough space to remap.
fn build_remaps(ranges: Vec<BlockRange>, free: Vec<BlockRange>) -> Vec<(BlockRange, BlockRange)> {
use std::cmp::Ordering;
let mut remap = Vec::new();
let mut range_iter = ranges.into_iter();
let mut free_iter = free.into_iter();
let mut r_ = range_iter.next();
let mut f_ = free_iter.next();
while let (Some(r), Some(f)) = (r_, f_) {
let rlen = range_len(&r);
let flen = range_len(&f);
match rlen.cmp(&flen) {
Ordering::Less => {
// range fits into the free chunk
remap.push((r, f.start..(f.start + rlen)));
f_ = Some((f.start + rlen)..f.end);
r_ = range_iter.next();
}
Ordering::Equal => {
remap.push((r, f));
r_ = range_iter.next();
f_ = free_iter.next();
}
Ordering::Greater => {
remap.push((r.start..(r.start + flen), f));
r_ = Some((r.start + flen)..r.end);
f_ = free_iter.next();
}
}
}
remap
}
#[test]
fn test_build_remaps() {
struct Test {
ranges: Vec<BlockRange>,
free: Vec<BlockRange>,
result: Vec<(BlockRange, BlockRange)>,
}
let tests = vec![
Test {
ranges: vec![],
free: vec![],
result: vec![],
},
Test {
ranges: vec![],
free: vec![0..100],
result: vec![],
},
Test {
ranges: vec![1000..1002],
free: vec![0..100],
result: vec![(1000..1002, 0..2)],
},
Test {
ranges: vec![1000..1002, 1100..1110],
free: vec![0..100],
result: vec![(1000..1002, 0..2), (1100..1110, 2..12)],
},
Test {
ranges: vec![100..120],
free: vec![0..5, 20..23, 30..50],
result: vec![(100..105, 0..5), (105..108, 20..23), (108..120, 30..42)],
},
];
for t in tests {
assert_eq!(build_remaps(t.ranges, t.free), t.result);
}
}
fn overlaps(r1: &BlockRange, r2: &BlockRange, index: usize) -> Option<usize> {
if r1.start >= r2.end {
return None;
}
if r2.start >= r1.end {
return None;
}
Some(index)
}
// Finds the index of the first entry that overlaps r.
fn find_first(r: &BlockRange, remaps: &[(BlockRange, BlockRange)]) -> Option<usize> {
if remaps.is_empty() {
return None;
}
match remaps.binary_search_by_key(&r.start, |(from, _)| from.start) {
Ok(n) => Some(n),
Err(n) => {
if n == 0 {
let (from, _) = &remaps[n];
overlaps(&r, &from, n)
} else if n == remaps.len() {
let (from, _) = &remaps[n - 1];
overlaps(&r, from, n - 1)
} else {
// Need to check the previous entry
let (from, _) = &remaps[n - 1];
overlaps(&r, &from, n - 1).or_else(|| {
let (from, _) = &remaps[n];
overlaps(&r, &from, n)
})
}
}
}
}
fn is_empty(r: &BlockRange) -> bool {
r.start == r.end
}
// remaps must be in sorted order by from.start.
fn remap(r: &BlockRange, remaps: &[(BlockRange, BlockRange)]) -> Vec<BlockRange> {
let mut remap = Vec::new();
let mut r = r.start..r.end;
if let Some(index) = find_first(&r, &remaps) {
let mut index = index;
loop {
let (from, to) = &remaps[index];
// There may be a prefix that doesn't overlap with 'from'
if r.start < from.start {
let len = u64::min(range_len(&r), from.start - r.start);
remap.push(r.start..(r.start + len));
r = (r.start + len)..r.end;
if is_empty(&r) {
break;
}
}
let to = (to.start + (r.start - from.start))..to.end;
let from = r.start..from.end;
let rlen = range_len(&r);
let flen = range_len(&from);
let len = u64::min(rlen, flen);
remap.push(to.start..(to.start + len));
r = (r.start + len)..r.end;
if is_empty(&r) {
break;
}
if len == flen {
index += 1;
}
if index == remaps.len() {
remap.push(r.start..r.end);
break;
}
}
} else {
remap.push(r.start..r.end);
}
remap
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn remap_test() {
struct Test {
remaps: Vec<(BlockRange, BlockRange)>,
input: BlockRange,
output: Vec<BlockRange>,
}
let tests = [
Test {
remaps: vec![],
input: 0..1,
output: vec![0..1],
},
Test {
remaps: vec![],
input: 100..1000,
output: vec![100..1000],
},
Test {
remaps: vec![(10..20, 110..120)],
input: 0..5,
output: vec![0..5],
},
Test {
remaps: vec![(10..20, 110..120)],
input: 10..20,
output: vec![110..120],
},
Test {
remaps: vec![(10..20, 110..120)],
input: 5..15,
output: vec![5..10, 110..115],
},
Test {
remaps: vec![(10..20, 110..120)],
input: 5..25,
output: vec![5..10, 110..120, 20..25],
},
Test {
remaps: vec![(10..20, 110..120)],
input: 15..25,
output: vec![115..120, 20..25],
},
Test {
remaps: vec![(10..20, 110..120)],
input: 25..35,
output: vec![25..35],
},
Test {
remaps: vec![(10..20, 110..120), (30..40, 230..240)],
input: 0..50,
output: vec![0..10, 110..120, 20..30, 230..240, 40..50],
},
];
for t in &tests {
let rs = remap(&t.input, &t.remaps);
assert_eq!(rs, t.output);
}
}
}
fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> Vec<Region> {
let mut rs = Vec::new();
for (from, to) in remaps {
rs.push(Region {
src: from.start * block_size,
dest: to.start * block_size,
len: range_len(&from) * block_size,
});
}
rs
}
fn process_xml<MV: xml::MetadataVisitor>(input_path: &Path, pass: &mut MV) -> Result<()> {
let input = OpenOptions::new()
.read(true)
.write(false)
.custom_flags(libc::O_EXCL)
.open(input_path)?;
xml::read(input, pass)?;
Ok(())
}
pub fn shrink(
input_path: &Path,
output_path: &Path,
data_path: &Path,
nr_blocks: u64,
do_copy: bool,
) -> Result<()> {
let mut pass1 = Pass1::new(nr_blocks);
eprint!("Reading xml...");
process_xml(input_path, &mut pass1)?;
eprintln!("done");
eprintln!("{} blocks need moving", pass1.nr_high_blocks);
let ranges = bits_to_ranges(&pass1.allocated_blocks);
let (below, above) = ranges_split(&ranges, nr_blocks);
let free = negate_ranges(&below, nr_blocks);
let free_blocks = ranges_total(&free);
eprintln!("{} free blocks.", free_blocks);
if free_blocks < pass1.nr_high_blocks {
return Err(anyhow!("Insufficient space"));
}
let remaps = build_remaps(above, free);
if do_copy {
let regions = build_copy_regions(&remaps, pass1.block_size.unwrap() as u64);
copier::copy(data_path, &regions)?;
} else {
eprintln!("skipping copy");
}
let output = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.open(output_path)?;
let mut pass2 = Pass2::new(output, nr_blocks, remaps);
eprint!("writing new xml...");
process_xml(input_path, &mut pass2)?;
eprintln!("done.");
Ok(())
}
//---------------------------------------

0
src/check.rs → src/thin/check.rs
View File

2
src/thin/mod.rs
View File

@@ -1 +1,3 @@
mod superblock;
pub mod check;
pub mod xml;

413
src/thin/xml.rs Normal file
View File

@@ -0,0 +1,413 @@
use anyhow::Result;
use std::{borrow::Cow, fmt::Display, io::prelude::*, io::BufReader, io::Write};
use quick_xml::events::attributes::Attribute;
use quick_xml::events::{BytesEnd, BytesStart, Event};
use quick_xml::{Reader, Writer};
//---------------------------------------
#[derive(Clone)]
pub struct Superblock {
pub uuid: String,
pub time: u64,
pub transaction: u64,
pub flags: Option<u32>,
pub version: Option<u32>,
pub data_block_size: u32,
pub nr_data_blocks: u64,
pub metadata_snap: Option<u64>,
}
#[derive(Clone)]
pub struct Device {
pub dev_id: u32,
pub mapped_blocks: u64,
pub transaction: u64,
pub creation_time: u64,
pub snap_time: u64,
}
#[derive(Clone)]
pub struct Map {
pub thin_begin: u64,
pub data_begin: u64,
pub time: u32,
pub len: u64,
}
#[derive(Clone)]
pub enum Visit {
Continue,
Stop,
}
pub trait MetadataVisitor {
fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit>;
fn superblock_e(&mut self) -> Result<Visit>;
fn device_b(&mut self, d: &Device) -> Result<Visit>;
fn device_e(&mut self) -> Result<Visit>;
fn map(&mut self, m: &Map) -> Result<Visit>;
fn eof(&mut self) -> Result<Visit>;
}
pub struct XmlWriter<W: Write> {
w: Writer<W>,
}
impl<W: Write> XmlWriter<W> {
pub fn new(w: W) -> XmlWriter<W> {
XmlWriter {
w: Writer::new_with_indent(w, 0x20, 2),
}
}
}
fn mk_attr_<'a, T: Display>(n: T) -> Cow<'a, [u8]> {
let str = format!("{}", n);
Cow::Owned(str.into_bytes())
}
fn mk_attr<T: Display>(key: &[u8], value: T) -> Attribute {
Attribute {
key,
value: mk_attr_(value),
}
}
const XML_VERSION: u32 = 2;
impl<W: Write> MetadataVisitor for XmlWriter<W> {
fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit> {
let tag = b"superblock";
let mut elem = BytesStart::owned(tag.to_vec(), tag.len());
elem.push_attribute(mk_attr(b"uuid", sb.uuid.clone()));
elem.push_attribute(mk_attr(b"time", sb.time));
elem.push_attribute(mk_attr(b"transaction", sb.transaction));
if let Some(flags) = sb.flags {
// FIXME: is this really a nr?
elem.push_attribute(mk_attr(b"flags", flags));
}
elem.push_attribute(mk_attr(b"version", XML_VERSION));
elem.push_attribute(mk_attr(b"data_block_size", sb.data_block_size));
elem.push_attribute(mk_attr(b"nr_data_blocks", sb.nr_data_blocks));
if let Some(snap) = sb.metadata_snap {
elem.push_attribute(mk_attr(b"metadata_snap", snap));
}
self.w.write_event(Event::Start(elem))?;
Ok(Visit::Continue)
}
fn superblock_e(&mut self) -> Result<Visit> {
self.w
.write_event(Event::End(BytesEnd::borrowed(b"superblock")))?;
Ok(Visit::Continue)
}
fn device_b(&mut self, d: &Device) -> Result<Visit> {
let tag = b"device";
let mut elem = BytesStart::owned(tag.to_vec(), tag.len());
elem.push_attribute(mk_attr(b"dev_id", d.dev_id));
elem.push_attribute(mk_attr(b"mapped_blocks", d.mapped_blocks));
elem.push_attribute(mk_attr(b"transaction", d.transaction));
elem.push_attribute(mk_attr(b"creation_time", d.creation_time));
elem.push_attribute(mk_attr(b"snap_time", d.snap_time));
self.w.write_event(Event::Start(elem))?;
Ok(Visit::Continue)
}
fn device_e(&mut self) -> Result<Visit> {
self.w
.write_event(Event::End(BytesEnd::borrowed(b"device")))?;
Ok(Visit::Continue)
}
fn map(&mut self, m: &Map) -> Result<Visit> {
match m.len {
1 => {
let tag = b"single_mapping";
let mut elem = BytesStart::owned(tag.to_vec(), tag.len());
elem.push_attribute(mk_attr(b"origin_block", m.thin_begin));
elem.push_attribute(mk_attr(b"data_block", m.data_begin));
elem.push_attribute(mk_attr(b"time", m.time));
self.w.write_event(Event::Empty(elem))?;
}
_ => {
let tag = b"range_mapping";
let mut elem = BytesStart::owned(tag.to_vec(), tag.len());
elem.push_attribute(mk_attr(b"origin_begin", m.thin_begin));
elem.push_attribute(mk_attr(b"data_begin", m.data_begin));
elem.push_attribute(mk_attr(b"length", m.len));
elem.push_attribute(mk_attr(b"time", m.time));
self.w.write_event(Event::Empty(elem))?;
}
}
Ok(Visit::Continue)
}
fn eof(&mut self) -> Result<Visit> {
let w = self.w.inner();
w.flush()?;
Ok(Visit::Continue)
}
}
//---------------------------------------
// FIXME: nasty unwraps
fn string_val(kv: &Attribute) -> String {
let v = kv.unescaped_value().unwrap();
let bytes = v.to_vec();
String::from_utf8(bytes).unwrap()
}
// FIXME: there's got to be a way of doing this without copying the string
fn u64_val(kv: &Attribute) -> Result<u64> {
let n = string_val(kv).parse::<u64>()?;
Ok(n)
}
fn u32_val(kv: &Attribute) -> Result<u32> {
let n = string_val(kv).parse::<u32>()?;
Ok(n)
}
fn bad_attr<T>(_tag: &str, _attr: &[u8]) -> Result<T> {
todo!();
}
fn missing_attr<T>(_tag: &str, _attr: &str) -> Result<T> {
todo!();
}
fn check_attr<T>(tag: &str, name: &str, maybe_v: Option<T>) -> Result<T> {
match maybe_v {
None => missing_attr(tag, name),
Some(v) => Ok(v),
}
}
fn parse_superblock(e: &BytesStart) -> Result<Superblock> {
let mut uuid: Option<String> = None;
let mut time: Option<u64> = None;
let mut transaction: Option<u64> = None;
let mut flags: Option<u32> = None;
let mut version: Option<u32> = None;
let mut data_block_size: Option<u32> = None;
let mut nr_data_blocks: Option<u64> = None;
let mut metadata_snap: Option<u64> = None;
for a in e.attributes() {
let kv = a.unwrap();
match kv.key {
b"uuid" => uuid = Some(string_val(&kv)),
b"time" => time = Some(u64_val(&kv)?),
b"transaction" => transaction = Some(u64_val(&kv)?),
b"flags" => flags = Some(u32_val(&kv)?),
b"version" => version = Some(u32_val(&kv)?),
b"data_block_size" => data_block_size = Some(u32_val(&kv)?),
b"nr_data_blocks" => nr_data_blocks = Some(u64_val(&kv)?),
b"metadata_snap" => metadata_snap = Some(u64_val(&kv)?),
_ => return bad_attr("superblock", kv.key),
}
}
let tag = "superblock";
Ok(Superblock {
uuid: check_attr(tag, "uuid", uuid)?,
time: check_attr(tag, "time", time)?,
transaction: check_attr(tag, "transaction", transaction)?,
flags,
version,
data_block_size: check_attr(tag, "data_block_size", data_block_size)?,
nr_data_blocks: check_attr(tag, "nr_data_blocks", nr_data_blocks)?,
metadata_snap,
})
}
fn parse_device(e: &BytesStart) -> Result<Device> {
let mut dev_id: Option<u32> = None;
let mut mapped_blocks: Option<u64> = None;
let mut transaction: Option<u64> = None;
let mut creation_time: Option<u64> = None;
let mut snap_time: Option<u64> = None;
for a in e.attributes() {
let kv = a.unwrap();
match kv.key {
b"dev_id" => dev_id = Some(u32_val(&kv)?),
b"mapped_blocks" => mapped_blocks = Some(u64_val(&kv)?),
b"transaction" => transaction = Some(u64_val(&kv)?),
b"creation_time" => creation_time = Some(u64_val(&kv)?),
b"snap_time" => snap_time = Some(u64_val(&kv)?),
_ => return bad_attr("device", kv.key),
}
}
let tag = "device";
Ok(Device {
dev_id: check_attr(tag, "dev_id", dev_id)?,
mapped_blocks: check_attr(tag, "mapped_blocks", mapped_blocks)?,
transaction: check_attr(tag, "transaction", transaction)?,
creation_time: check_attr(tag, "creation_time", creation_time)?,
snap_time: check_attr(tag, "snap_time", snap_time)?,
})
}
fn parse_single_map(e: &BytesStart) -> Result<Map> {
let mut thin_begin: Option<u64> = None;
let mut data_begin: Option<u64> = None;
let mut time: Option<u32> = None;
for a in e.attributes() {
let kv = a.unwrap();
match kv.key {
b"origin_block" => thin_begin = Some(u64_val(&kv)?),
b"data_block" => data_begin = Some(u64_val(&kv)?),
b"time" => time = Some(u32_val(&kv)?),
_ => return bad_attr("single_mapping", kv.key),
}
}
let tag = "single_mapping";
Ok(Map {
thin_begin: check_attr(tag, "origin_block", thin_begin)?,
data_begin: check_attr(tag, "data_block", data_begin)?,
time: check_attr(tag, "time", time)?,
len: 1,
})
}
fn parse_range_map(e: &BytesStart) -> Result<Map> {
let mut thin_begin: Option<u64> = None;
let mut data_begin: Option<u64> = None;
let mut time: Option<u32> = None;
let mut length: Option<u64> = None;
for a in e.attributes() {
let kv = a.unwrap();
match kv.key {
b"origin_begin" => thin_begin = Some(u64_val(&kv)?),
b"data_begin" => data_begin = Some(u64_val(&kv)?),
b"time" => time = Some(u32_val(&kv)?),
b"length" => length = Some(u64_val(&kv)?),
_ => return bad_attr("range_mapping", kv.key),
}
}
let tag = "range_mapping";
Ok(Map {
thin_begin: check_attr(tag, "origin_begin", thin_begin)?,
data_begin: check_attr(tag, "data_begin", data_begin)?,
time: check_attr(tag, "time", time)?,
len: check_attr(tag, "length", length)?,
})
}
fn handle_event<R, M>(reader: &mut Reader<R>, buf: &mut Vec<u8>, visitor: &mut M) -> Result<Visit>
where
R: Read + BufRead,
M: MetadataVisitor,
{
match reader.read_event(buf) {
Ok(Event::Start(ref e)) => match e.name() {
b"superblock" => visitor.superblock_b(&parse_superblock(e)?),
b"device" => visitor.device_b(&parse_device(e)?),
_ => todo!(),
},
Ok(Event::End(ref e)) => match e.name() {
b"superblock" => visitor.superblock_e(),
b"device" => visitor.device_e(),
_ => todo!(),
},
Ok(Event::Empty(ref e)) => match e.name() {
b"single_mapping" => visitor.map(&parse_single_map(e)?),
b"range_mapping" => visitor.map(&parse_range_map(e)?),
_ => todo!(),
},
Ok(Event::Text(_)) => Ok(Visit::Continue),
Ok(Event::Comment(_)) => Ok(Visit::Continue),
Ok(Event::Eof) => {
visitor.eof()?;
Ok(Visit::Stop)
}
Ok(_) => todo!(),
// FIXME: don't panic!
Err(e) => panic!("error parsing xml {:?}", e),
}
}
pub fn read<R, M>(input: R, visitor: &mut M) -> Result<()>
where
R: Read,
M: MetadataVisitor,
{
let input = BufReader::new(input);
let mut reader = Reader::from_reader(input);
reader.trim_text(true);
let mut buf = Vec::new();
loop {
match handle_event(&mut reader, &mut buf, visitor)? {
Visit::Continue => {}
Visit::Stop => break,
}
}
Ok(())
}
//---------------------------------------
struct SBVisitor {
superblock: Option<Superblock>,
}
impl MetadataVisitor for SBVisitor {
fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit> {
self.superblock = Some(sb.clone());
Ok(Visit::Stop)
}
fn superblock_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_b(&mut self, _d: &Device) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn map(&mut self, _m: &Map) -> Result<Visit> {
Ok(Visit::Continue)
}
fn eof(&mut self) -> Result<Visit> {
Ok(Visit::Stop)
}
}
pub fn read_superblock<R>(input: R) -> Result<Superblock>
where
R: Read,
{
let mut v = SBVisitor {superblock: None};
read(input, &mut v)?;
Ok(v.superblock.unwrap())
}
//---------------------------------------