[thin_shrink] write test harness

This commit is contained in:
Joe Thornber 2020-06-26 16:44:47 +01:00
parent 07da5704d5
commit d03dac8f75
9 changed files with 458 additions and 90 deletions

30
Cargo.lock generated
View File

@ -324,6 +324,11 @@ dependencies = [
"rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "redox_syscall"
version = "0.1.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.3.9" version = "1.3.9"
@ -340,6 +345,14 @@ name = "regex-syntax"
version = "0.6.18" version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.2.3" version = "0.2.3"
@ -386,6 +399,19 @@ dependencies = [
"unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "tempfile"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
"remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.11.0" version = "0.11.0"
@ -414,6 +440,7 @@ dependencies = [
"quickcheck 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
"quickcheck_macros 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck_macros 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -515,8 +542,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" "checksum rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" "checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" "checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
"checksum regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" "checksum regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
"checksum regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)" = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" "checksum regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)" = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
"checksum remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" "checksum ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
@ -524,6 +553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" "checksum static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3"
"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
"checksum syn 1.0.30 (registry+https://github.com/rust-lang/crates.io-index)" = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2" "checksum syn 1.0.30 (registry+https://github.com/rust-lang/crates.io-index)" = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2"
"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" "checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"

View File

@ -18,6 +18,7 @@ nix = "0.17"
nom = "5.1" nom = "5.1"
num_cpus = "1.13" num_cpus = "1.13"
rand = "0.7" rand = "0.7"
tempfile = "3.1"
num-traits = "0.2" num-traits = "0.2"
num-derive = "0.3" num-derive = "0.3"

View File

@ -1,8 +1,10 @@
use nix::sys::stat; use nix::sys::stat;
use nix::sys::stat::{FileStat, SFlag}; use nix::sys::stat::{FileStat, SFlag};
use std::io;
use std::fs::File; use std::fs::File;
use std::io;
use std::io::{Seek, Write};
use std::os::unix::io::AsRawFd; use std::os::unix::io::AsRawFd;
use tempfile::tempfile;
//--------------------------------------- //---------------------------------------
@ -11,15 +13,13 @@ fn check_bits(mode: u32, flag: &SFlag) -> bool {
} }
pub fn is_file_or_blk(info: FileStat) -> bool { pub fn is_file_or_blk(info: FileStat) -> bool {
check_bits(info.st_mode, &stat::SFlag::S_IFBLK) || check_bits(info.st_mode, &stat::SFlag::S_IFBLK)
check_bits(info.st_mode, &stat::SFlag::S_IFREG) || check_bits(info.st_mode, &stat::SFlag::S_IFREG)
} }
pub fn file_exists(path: &str) -> bool { pub fn file_exists(path: &str) -> bool {
match stat::stat(path) { match stat::stat(path) {
Ok(info) => { Ok(info) => is_file_or_blk(info),
is_file_or_blk(info)
}
_ => { _ => {
// FIXME: assuming all errors indicate the file doesn't // FIXME: assuming all errors indicate the file doesn't
// exist. // exist.
@ -40,14 +40,14 @@ pub fn fail<T>(msg: &str) -> io::Result<T> {
} }
fn get_device_size(path: &str) -> io::Result<u64> { fn get_device_size(path: &str) -> io::Result<u64> {
let file = File::open(path)?; let file = File::open(path)?;
let fd = file.as_raw_fd(); let fd = file.as_raw_fd();
let mut cap = 0u64; let mut cap = 0u64;
unsafe { unsafe {
match ioctl_blkgetsize64(fd, &mut cap) { match ioctl_blkgetsize64(fd, &mut cap) {
Ok(_) => {Ok(cap)} Ok(_) => Ok(cap),
_ => {fail("BLKGETSIZE64 ioctl failed")} _ => fail("BLKGETSIZE64 ioctl failed"),
} }
} }
} }
@ -60,12 +60,25 @@ pub fn file_size(path: &str) -> io::Result<u64> {
get_device_size(path) get_device_size(path)
} else { } else {
fail("not a regular file or block device") fail("not a regular file or block device")
} }
} }
_ => { _ => fail("stat failed"),
fail("stat failed") }
} }
}
//---------------------------------------
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = tempfile()?;
let zeroes: Vec<u8> = vec![0; 1];
if nr_bytes > 0 {
file.seek(io::SeekFrom::Start(nr_bytes - 1))?;
file.write_all(&zeroes)?;
}
Ok(file)
} }
//--------------------------------------- //---------------------------------------

View File

@ -20,4 +20,5 @@ pub mod check;
pub mod file_utils; pub mod file_utils;
pub mod pack; pub mod pack;
pub mod shrink; pub mod shrink;
pub mod thin;
pub mod version; pub mod version;

View File

@ -1,4 +1,3 @@
pub mod toplevel; pub mod toplevel;
mod copier; mod copier;
mod xml;

View File

@ -5,7 +5,7 @@ use std::io::Write;
use std::os::unix::fs::OpenOptionsExt; use std::os::unix::fs::OpenOptionsExt;
use crate::shrink::copier::{self, Region}; use crate::shrink::copier::{self, Region};
use crate::shrink::xml; use crate::thin::xml::{self, Visit};
//--------------------------------------- //---------------------------------------
@ -34,36 +34,36 @@ impl Pass1 {
} }
impl xml::MetadataVisitor for Pass1 { impl xml::MetadataVisitor for Pass1 {
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<()> { fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
self.allocated_blocks.grow(sb.nr_data_blocks as usize); self.allocated_blocks.grow(sb.nr_data_blocks as usize);
self.block_size = Some(sb.data_block_size as u64); self.block_size = Some(sb.data_block_size as u64);
Ok(()) Ok(Visit::Continue)
} }
fn superblock_e(&mut self) -> Result<()> { fn superblock_e(&mut self) -> Result<Visit> {
Ok(()) Ok(Visit::Continue)
} }
fn device_b(&mut self, _d: &xml::Device) -> Result<()> { fn device_b(&mut self, _d: &xml::Device) -> Result<Visit> {
Ok(()) Ok(Visit::Continue)
} }
fn device_e(&mut self) -> Result<()> { fn device_e(&mut self) -> Result<Visit> {
Ok(()) Ok(Visit::Continue)
} }
fn map(&mut self, m: &xml::Map) -> Result<()> { fn map(&mut self, m: &xml::Map) -> Result<Visit> {
for i in m.data_begin..(m.data_begin + m.len) { for i in m.data_begin..(m.data_begin + m.len) {
if i > self.nr_blocks { if i > self.nr_blocks {
self.nr_high_blocks += 1; self.nr_high_blocks += 1;
} }
self.allocated_blocks.insert(i as usize); self.allocated_blocks.insert(i as usize);
} }
Ok(()) Ok(Visit::Continue)
} }
fn eof(&mut self) -> Result<()> { fn eof(&mut self) -> Result<Visit> {
Ok(()) Ok(Visit::Continue)
} }
} }
@ -87,23 +87,23 @@ impl<W: Write> Pass2<W> {
} }
impl<W: Write> xml::MetadataVisitor for Pass2<W> { impl<W: Write> xml::MetadataVisitor for Pass2<W> {
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<()> { fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
self.writer.superblock_b(sb) self.writer.superblock_b(sb)
} }
fn superblock_e(&mut self) -> Result<()> { fn superblock_e(&mut self) -> Result<Visit> {
self.writer.superblock_e() self.writer.superblock_e()
} }
fn device_b(&mut self, d: &xml::Device) -> Result<()> { fn device_b(&mut self, d: &xml::Device) -> Result<Visit> {
self.writer.device_b(d) self.writer.device_b(d)
} }
fn device_e(&mut self) -> Result<()> { fn device_e(&mut self) -> Result<Visit> {
self.writer.device_e() self.writer.device_e()
} }
fn map(&mut self, m: &xml::Map) -> Result<()> { fn map(&mut self, m: &xml::Map) -> Result<Visit> {
if m.data_begin + m.len < self.nr_blocks { if m.data_begin + m.len < self.nr_blocks {
// no remapping needed. // no remapping needed.
self.writer.map(m)?; self.writer.map(m)?;
@ -123,10 +123,10 @@ impl<W: Write> xml::MetadataVisitor for Pass2<W> {
} }
} }
Ok(()) Ok(Visit::Continue)
} }
fn eof(&mut self) -> Result<()> { fn eof(&mut self) -> Result<Visit> {
self.writer.eof() self.writer.eof()
} }
} }
@ -181,7 +181,7 @@ fn ranges_split(ranges: &[BlockRange], threshold: u64) -> (Vec<BlockRange>, Vec<
(below, above) (below, above)
} }
fn negate_ranges(ranges: &[BlockRange]) -> Vec<BlockRange> { fn negate_ranges(ranges: &[BlockRange], upper_limit: u64) -> Vec<BlockRange> {
use std::ops::Range; use std::ops::Range;
let mut result = Vec::new(); let mut result = Vec::new();
@ -199,6 +199,10 @@ fn negate_ranges(ranges: &[BlockRange]) -> Vec<BlockRange> {
} }
} }
if cursor < upper_limit {
result.push(cursor..upper_limit);
}
result result
} }
@ -443,7 +447,7 @@ pub fn shrink(input_path: &str, output_path: &str, data_path: &str, nr_blocks: u
let ranges = bits_to_ranges(&pass1.allocated_blocks); let ranges = bits_to_ranges(&pass1.allocated_blocks);
let (below, above) = ranges_split(&ranges, nr_blocks); let (below, above) = ranges_split(&ranges, nr_blocks);
let free = negate_ranges(&below); let free = negate_ranges(&below, nr_blocks);
let free_blocks = ranges_total(&free); let free_blocks = ranges_total(&free);
eprintln!("{} free blocks.", free_blocks); eprintln!("{} free blocks.", free_blocks);

1
src/thin/mod.rs Normal file
View File

@ -0,0 +1 @@
pub mod xml;

View File

@ -1,11 +1,5 @@
use anyhow::Result; use anyhow::Result;
use std::{ use std::{borrow::Cow, fmt::Display, io::prelude::*, io::BufReader, io::Write};
borrow::{Cow},
fmt::Display,
io::prelude::*,
io::BufReader,
io::Write,
};
use quick_xml::events::attributes::Attribute; use quick_xml::events::attributes::Attribute;
use quick_xml::events::{BytesEnd, BytesStart, Event}; use quick_xml::events::{BytesEnd, BytesStart, Event};
@ -13,6 +7,7 @@ use quick_xml::{Reader, Writer};
//--------------------------------------- //---------------------------------------
#[derive(Clone)]
pub struct Superblock { pub struct Superblock {
pub uuid: String, pub uuid: String,
pub time: u64, pub time: u64,
@ -24,6 +19,7 @@ pub struct Superblock {
pub metadata_snap: Option<u64>, pub metadata_snap: Option<u64>,
} }
#[derive(Clone)]
pub struct Device { pub struct Device {
pub dev_id: u32, pub dev_id: u32,
pub mapped_blocks: u64, pub mapped_blocks: u64,
@ -32,6 +28,7 @@ pub struct Device {
pub snap_time: u64, pub snap_time: u64,
} }
#[derive(Clone)]
pub struct Map { pub struct Map {
pub thin_begin: u64, pub thin_begin: u64,
pub data_begin: u64, pub data_begin: u64,
@ -39,16 +36,22 @@ pub struct Map {
pub len: u64, pub len: u64,
} }
#[derive(Clone)]
pub enum Visit {
Continue,
Stop,
}
pub trait MetadataVisitor { pub trait MetadataVisitor {
fn superblock_b(&mut self, sb: &Superblock) -> Result<()>; fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit>;
fn superblock_e(&mut self) -> Result<()>; fn superblock_e(&mut self) -> Result<Visit>;
fn device_b(&mut self, d: &Device) -> Result<()>; fn device_b(&mut self, d: &Device) -> Result<Visit>;
fn device_e(&mut self) -> Result<()>; fn device_e(&mut self) -> Result<Visit>;
fn map(&mut self, m: &Map) -> Result<()>; fn map(&mut self, m: &Map) -> Result<Visit>;
fn eof(&mut self) -> Result<()>; fn eof(&mut self) -> Result<Visit>;
} }
pub struct XmlWriter<W: Write> { pub struct XmlWriter<W: Write> {
@ -57,7 +60,9 @@ pub struct XmlWriter<W: Write> {
impl<W: Write> XmlWriter<W> { impl<W: Write> XmlWriter<W> {
pub fn new(w: W) -> XmlWriter<W> { pub fn new(w: W) -> XmlWriter<W> {
XmlWriter { w: Writer::new_with_indent(w, 0x20, 2) } XmlWriter {
w: Writer::new_with_indent(w, 0x20, 2),
}
} }
} }
@ -76,7 +81,7 @@ fn mk_attr<T: Display>(key: &[u8], value: T) -> Attribute {
const XML_VERSION: u32 = 2; const XML_VERSION: u32 = 2;
impl<W: Write> MetadataVisitor for XmlWriter<W> { impl<W: Write> MetadataVisitor for XmlWriter<W> {
fn superblock_b(&mut self, sb: &Superblock) -> Result<()> { fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit> {
let tag = b"superblock"; let tag = b"superblock";
let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); let mut elem = BytesStart::owned(tag.to_vec(), tag.len());
elem.push_attribute(mk_attr(b"uuid", sb.uuid.clone())); elem.push_attribute(mk_attr(b"uuid", sb.uuid.clone()));
@ -96,16 +101,16 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
} }
self.w.write_event(Event::Start(elem))?; self.w.write_event(Event::Start(elem))?;
Ok(()) Ok(Visit::Continue)
} }
fn superblock_e(&mut self) -> Result<()> { fn superblock_e(&mut self) -> Result<Visit> {
self.w self.w
.write_event(Event::End(BytesEnd::borrowed(b"superblock")))?; .write_event(Event::End(BytesEnd::borrowed(b"superblock")))?;
Ok(()) Ok(Visit::Continue)
} }
fn device_b(&mut self, d: &Device) -> Result<()> { fn device_b(&mut self, d: &Device) -> Result<Visit> {
let tag = b"device"; let tag = b"device";
let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); let mut elem = BytesStart::owned(tag.to_vec(), tag.len());
elem.push_attribute(mk_attr(b"dev_id", d.dev_id)); elem.push_attribute(mk_attr(b"dev_id", d.dev_id));
@ -114,16 +119,16 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
elem.push_attribute(mk_attr(b"creation_time", d.creation_time)); elem.push_attribute(mk_attr(b"creation_time", d.creation_time));
elem.push_attribute(mk_attr(b"snap_time", d.snap_time)); elem.push_attribute(mk_attr(b"snap_time", d.snap_time));
self.w.write_event(Event::Start(elem))?; self.w.write_event(Event::Start(elem))?;
Ok(()) Ok(Visit::Continue)
} }
fn device_e(&mut self) -> Result<()> { fn device_e(&mut self) -> Result<Visit> {
self.w self.w
.write_event(Event::End(BytesEnd::borrowed(b"device")))?; .write_event(Event::End(BytesEnd::borrowed(b"device")))?;
Ok(()) Ok(Visit::Continue)
} }
fn map(&mut self, m: &Map) -> Result<()> { fn map(&mut self, m: &Map) -> Result<Visit> {
match m.len { match m.len {
1 => { 1 => {
let tag = b"single_mapping"; let tag = b"single_mapping";
@ -143,13 +148,13 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
self.w.write_event(Event::Empty(elem))?; self.w.write_event(Event::Empty(elem))?;
} }
} }
Ok(()) Ok(Visit::Continue)
} }
fn eof(&mut self) -> Result<()> { fn eof(&mut self) -> Result<Visit> {
let w = self.w.inner(); let w = self.w.inner();
w.flush()?; w.flush()?;
Ok(()) Ok(Visit::Continue)
} }
} }
@ -184,7 +189,7 @@ fn missing_attr<T>(_tag: &str, _attr: &str) -> Result<T> {
fn check_attr<T>(tag: &str, name: &str, maybe_v: Option<T>) -> Result<T> { fn check_attr<T>(tag: &str, name: &str, maybe_v: Option<T>) -> Result<T> {
match maybe_v { match maybe_v {
None => missing_attr(tag, name), None => missing_attr(tag, name),
Some(v) => Ok(v) Some(v) => Ok(v),
} }
} }
@ -278,7 +283,7 @@ fn parse_single_map(e: &BytesStart) -> Result<Map> {
thin_begin: check_attr(tag, "origin_block", thin_begin)?, thin_begin: check_attr(tag, "origin_block", thin_begin)?,
data_begin: check_attr(tag, "data_block", data_begin)?, data_begin: check_attr(tag, "data_block", data_begin)?,
time: check_attr(tag, "time", time)?, time: check_attr(tag, "time", time)?,
len: 1 len: 1,
}) })
} }
@ -309,6 +314,40 @@ fn parse_range_map(e: &BytesStart) -> Result<Map> {
}) })
} }
fn handle_event<R, M>(reader: &mut Reader<R>, buf: &mut Vec<u8>, visitor: &mut M) -> Result<Visit>
where
R: Read + BufRead,
M: MetadataVisitor,
{
match reader.read_event(buf) {
Ok(Event::Start(ref e)) => match e.name() {
b"superblock" => visitor.superblock_b(&parse_superblock(e)?),
b"device" => visitor.device_b(&parse_device(e)?),
_ => todo!(),
},
Ok(Event::End(ref e)) => match e.name() {
b"superblock" => visitor.superblock_e(),
b"device" => visitor.device_e(),
_ => todo!(),
},
Ok(Event::Empty(ref e)) => match e.name() {
b"single_mapping" => visitor.map(&parse_single_map(e)?),
b"range_mapping" => visitor.map(&parse_range_map(e)?),
_ => todo!(),
},
Ok(Event::Text(_)) => Ok(Visit::Continue),
Ok(Event::Comment(_)) => Ok(Visit::Continue),
Ok(Event::Eof) => {
visitor.eof()?;
Ok(Visit::Stop)
}
Ok(_) => todo!(),
// FIXME: don't panic!
Err(e) => panic!("error parsing xml {:?}", e),
}
}
pub fn read<R, M>(input: R, visitor: &mut M) -> Result<()> pub fn read<R, M>(input: R, visitor: &mut M) -> Result<()>
where where
R: Read, R: Read,
@ -321,29 +360,9 @@ where
let mut buf = Vec::new(); let mut buf = Vec::new();
loop { loop {
match reader.read_event(&mut buf) { match handle_event(&mut reader, &mut buf, visitor)? {
Ok(Event::Start(ref e)) => match e.name() { Visit::Continue => {}
b"superblock" => visitor.superblock_b(&parse_superblock(e)?)?, Visit::Stop => break,
b"device" => visitor.device_b(&parse_device(e)?)?,
_ => todo!(),
},
Ok(Event::End(ref e)) => match e.name() {
b"superblock" => visitor.superblock_e()?,
b"device" => visitor.device_e()?,
_ => todo!(),
},
Ok(Event::Empty(ref e)) => match e.name() {
b"single_mapping" => visitor.map(&parse_single_map(e)?)?,
b"range_mapping" => visitor.map(&parse_range_map(e)?)?,
_ => todo!(),
},
Ok(Event::Text(_)) => {}
Ok(Event::Comment(_)) => {}
Ok(Event::Eof) => break,
Ok(_) => todo!(),
// FIXME: don't panic!
Err(e) => panic!("error parsing xml {:?}", e),
} }
} }
@ -351,3 +370,44 @@ where
} }
//--------------------------------------- //---------------------------------------
struct SBVisitor {
superblock: Option<Superblock>,
}
impl MetadataVisitor for SBVisitor {
fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit> {
self.superblock = Some(sb.clone());
Ok(Visit::Stop)
}
fn superblock_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_b(&mut self, _d: &Device) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn map(&mut self, _m: &Map) -> Result<Visit> {
Ok(Visit::Continue)
}
fn eof(&mut self) -> Result<Visit> {
Ok(Visit::Stop)
}
}
pub fn read_superblock<R>(input: R) -> Result<Superblock>
where
R: Read,
{
let mut v = SBVisitor {superblock: None};
read(input, &mut v)?;
Ok(v.superblock.unwrap())
}
//---------------------------------------

259
tests/thin_shrink.rs Normal file
View File

@ -0,0 +1,259 @@
use anyhow::Result;
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::Rng;
use std::fs::OpenOptions;
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
use thinp::file_utils;
use thinp::thin::xml::{self, Visit};
//------------------------------------
struct ThinBlock {
thin_id: u32,
thin_block: u64,
data_block: u64,
block_size: usize,
}
struct ThinReadRef {
pub data: Vec<u8>,
}
struct ThinWriteRef<'a, W: Write + Seek> {
file: &'a mut W,
pub data: Vec<u8>,
}
impl ThinBlock {
fn read_ref<R: Read + Seek>(&self, r: &mut R) -> Result<ThinReadRef> {
let mut rr = ThinReadRef {
data: vec![0; self.block_size],
};
r.seek(SeekFrom::Start(self.data_block * (self.block_size as u64)))?;
r.read_exact(&mut rr.data[0..])?;
Ok(rr)
}
fn zero_ref<'a, W: Write + Seek>(&self, w: &'a mut W) -> ThinWriteRef<'a, W> {
ThinWriteRef {
file: w,
data: vec![0; self.block_size],
}
}
fn write_ref<'a, W>(&self, w: &'a mut W) -> Result<ThinWriteRef<'a, W>>
where
W: Read + Write + Seek,
{
let mut data = vec![0; self.block_size];
w.seek(SeekFrom::Start(self.data_block * (self.block_size as u64)))?;
w.read_exact(&mut data[0..])?;
let wr = ThinWriteRef {
file: w,
data: vec![0; self.block_size],
};
Ok(wr)
}
}
impl<'a, W: Write + Seek> Drop for ThinWriteRef<'a, W> {
fn drop(&mut self) {
self.file.write_all(&self.data[0..]).unwrap();
}
}
//------------------------------------
trait ThinVisitor {
fn thin_block(&mut self, tb: &ThinBlock) -> Result<()>;
}
struct ThinXmlVisitor<'a, V: ThinVisitor> {
inner: &'a mut V,
block_size: Option<u32>,
thin_id: Option<u32>,
}
impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> {
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
self.block_size = Some(sb.data_block_size);
Ok(Visit::Continue)
}
fn superblock_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn device_b(&mut self, d: &xml::Device) -> Result<Visit> {
self.thin_id = Some(d.dev_id);
Ok(Visit::Continue)
}
fn device_e(&mut self) -> Result<Visit> {
Ok(Visit::Continue)
}
fn map(&mut self, m: &xml::Map) -> Result<Visit> {
for i in 0..m.len {
let block = ThinBlock {
thin_id: self.thin_id.unwrap(),
thin_block: m.thin_begin + i,
data_block: m.data_begin + i,
block_size: self.block_size.unwrap() as usize,
};
self.inner.thin_block(&block)?;
}
Ok(Visit::Continue)
}
fn eof(&mut self) -> Result<Visit> {
Ok(Visit::Stop)
}
}
fn thin_visit<R, M>(input: R, visitor: &mut M) -> Result<()>
where
R: Read,
M: ThinVisitor,
{
let mut xml_visitor = ThinXmlVisitor {
inner: visitor,
block_size: None,
thin_id: None,
};
xml::read(input, &mut xml_visitor)
}
//------------------------------------
// To test thin_shrink we'd like to stamp a known pattern across the
// provisioned areas of the thins in the pool, do the shrink, verify
// the patterns.
// A simple linear congruence generator used to create the data to
// go into the thin blocks.
struct Generator {
x: u64,
a: u64,
c: u64,
}
impl Generator {
fn new() -> Generator {
Generator {
x: 0,
a: 6364136223846793005,
c: 1442695040888963407,
}
}
fn step(&mut self) {
self.x = (self.a * self.x) + self.c
}
fn fill_buffer(&mut self, seed: u64, bytes: &mut [u8]) {
self.x = seed;
assert!(bytes.len() % 8 == 64);
let nr_words = bytes.len() / 8;
let mut out = Cursor::new(bytes);
for _ in 0..nr_words {
out.write_u64::<LittleEndian>(self.x).unwrap();
self.step();
}
}
fn verify_buffer(&mut self, seed: u64, bytes: &[u8]) {
self.x = seed;
assert!(bytes.len() % 8 == 64);
let nr_words = bytes.len() / 8;
let mut input = Cursor::new(bytes);
for _ in 0..nr_words {
let w = input.read_u64::<LittleEndian>().unwrap();
assert_eq!(w, self.x);
self.step();
}
}
}
//------------------------------------
struct Stamper<'a, W: Write + Seek> {
data_file: &'a mut W,
seed: u64,
}
impl<'a, W: Write + Seek> Stamper<'a, W> {
fn new(w: &'a mut W, seed: u64) -> Stamper<'a, W> {
Stamper { data_file: w, seed }
}
}
impl<'a, W: Write + Seek> ThinVisitor for Stamper<'a, W> {
fn thin_block(&mut self, b: &ThinBlock) -> Result<()> {
let mut wr = b.zero_ref(self.data_file);
let mut gen = Generator::new();
gen.fill_buffer(
self.seed ^ (b.thin_id as u64) ^ b.thin_block,
&mut wr.data[0..],
);
Ok(())
}
}
//------------------------------------
struct Verifier<'a, R: Read + Seek> {
data_file: &'a mut R,
seed: u64,
}
impl<'a, R: Read + Seek> Verifier<'a, R> {
fn new(r: &'a mut R, seed: u64) -> Verifier<'a, R> {
Verifier { data_file: r, seed }
}
}
impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> {
fn thin_block(&mut self, b: &ThinBlock) -> Result<()> {
let rr = b.read_ref(self.data_file)?;
let mut gen = Generator::new();
gen.verify_buffer(self.seed ^ (b.thin_id as u64) ^ b.thin_block, &rr.data[0..]);
Ok(())
}
}
//------------------------------------
fn create_data_file(xml_path: &str) -> Result<std::fs::File> {
let input = OpenOptions::new().read(true).write(false).open(xml_path)?;
let sb = xml::read_superblock(input)?;
let nr_blocks = sb.nr_data_blocks as u64;
let block_size = sb.data_block_size as u64 * 512;
let file = file_utils::temp_file_sized(nr_blocks * block_size)?;
Ok(file)
}
fn main(xml_path: &str) -> Result<()> {
let mut data_file = create_data_file(xml_path)?;
let mut xml_in = OpenOptions::new().read(true).write(false).open(xml_path)?;
let mut rng = rand::thread_rng();
let seed = rng.gen::<u64>();
let mut stamper = Stamper::new(&mut data_file, seed);
thin_visit(&mut xml_in, &mut stamper)?;
let mut verifier = Verifier::new(&mut data_file, seed);
thin_visit(&mut xml_in, &mut verifier)?;
Ok(())
}