From fcfcc60b89919afd7a1e852150b6be341ce40b26 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 31 Jul 2020 11:04:12 +0100 Subject: [PATCH] [functional-tests] Move thin_check functional tests to Rust. They'll be run as part of 'cargo test' now. --- Cargo.lock | 40 ++ Cargo.toml | 2 + functional-tests/thin-functional-tests.scm | 80 --- tests/common/mod.rs | 11 + tests/common/xml_generator.rs | 539 +++++++++++++++++++++ tests/thin_check.rs | 173 +++++++ tests/thin_shrink.rs | 529 +------------------- 7 files changed, 776 insertions(+), 598 deletions(-) create mode 100644 tests/common/mod.rs create mode 100644 tests/common/xml_generator.rs create mode 100644 tests/thin_check.rs diff --git a/Cargo.lock b/Cargo.lock index a41568a..9ec72e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,6 +107,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "duct" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90a9c3a25aafbd538c7d40a53f83c4487ee8216c12d1c8ef2c01eb2f6ea1553" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -260,6 +272,12 @@ dependencies = [ "libc", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + [[package]] name = "lazy_static" version = "1.4.0" @@ -369,6 +387,16 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +[[package]] +name = "os_pipe" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "pin-project" version = "0.4.23" @@ -543,6 +571,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "shared_child" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cebcf3a403e4deafaf34dc882c4a1b6a648b43e5670aa2e4bb985914eaeb2d2" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "slab" version = "0.4.2" @@ -603,10 +641,12 @@ dependencies = [ "byteorder", "clap", "crc32c", + "duct", "fixedbitset", "flate2", "futures", "io-uring", + "json", "libc", "nix", "nom", diff --git a/Cargo.toml b/Cargo.toml index e4c1fd1..dbb8470 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0" byteorder = "1.3" clap = "2.33" crc32c = "0.4" +duct = "0.13" fixedbitset = "0.3" futures = "0.3" flate2 = "1.0" @@ -27,5 +28,6 @@ threadpool = "1.8" thiserror = "1.0" [dev-dependencies] +json = "0.12" quickcheck = "0.9" quickcheck_macros = "0.9" diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index d5e5d00..5202d94 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,86 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_check scenarios - ;;;----------------------------------------------------------- - - (define-scenario (thin-check v) - "thin_check -V" - (run-ok-rcv (stdout _) (thin-check "-V") - (assert-equal tools-version stdout))) - - (define-scenario (thin-check version) - "thin_check --version" - (run-ok-rcv (stdout _) (thin-check "--version") - (assert-equal tools-version stdout))) - - (define-scenario (thin-check h) - "print help (-h)" - (run-ok-rcv (stdout _) (thin-check "-h") - (assert-equal thin-check-help stdout))) - - (define-scenario (thin-check help) - "print help (--help)" - (run-ok-rcv (stdout _) (thin-check "--help") - (assert-equal thin-check-help stdout))) - - (define-scenario (thin-check bad-option) - "Unrecognised option should cause failure" - (run-fail (thin-check "--hedgehogs-only"))) - - (define-scenario (thin-check superblock-only-valid) - "--super-block-only check passes on valid metadata" - (with-valid-metadata (md) - (run-ok (thin-check "--super-block-only" md)))) - - (define-scenario (thin-check superblock-only-invalid) - "--super-block-only check fails with corrupt metadata" - (with-corrupt-metadata (md) - (run-fail (thin-check "--super-block-only" md)))) - - (define-scenario (thin-check skip-mappings-valid) - "--skip-mappings check passes on valid metadata" - (with-valid-metadata (md) - (run-ok (thin-check "--skip-mappings" md)))) - - (define-scenario (thin-check ignore-non-fatal-errors) - "--ignore-non-fatal-errors check passes on valid metadata" - (with-valid-metadata (md) - (run-ok (thin-check "--ignore-non-fatal-errors" md)))) - - (define-scenario (thin-check quiet) - "--quiet should give no output" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-check "--quiet" md) - (assert-eof stdout) - (assert-eof stderr)))) - - (define-scenario (thin-check clear-needs-check-flag) - "Accepts --clear-needs-check-flag" - (with-valid-metadata (md) - (run-ok (thin-check "--clear-needs-check-flag" md)))) - - (define-scenario (thin-check tiny-metadata) - "Prints helpful message in case tiny metadata given" - (with-temp-file-sized ((md "thin.bin" 1024)) - (run-fail-rcv (_ stderr) (thin-check md) - (assert-starts-with "Metadata device/file too small. Is this binary metadata?" stderr)))) - - (define-scenario (thin-check spot-accidental-xml-data) - "Prints helpful message if XML metadata given" - (with-thin-xml (xml) - (system (fmt #f "man bash >> " xml)) - (run-fail-rcv (_ stderr) (thin-check xml) - (assert-matches ".*This looks like XML. thin_check only checks the binary metadata format." stderr)))) - - (define-scenario (thin-check info-fields) - "Outputs info fields" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-check md) - (assert-matches ".*TRANSACTION_ID=[0-9]+.*" stdout) - (assert-matches ".*METADATA_FREE_BLOCKS=[0-9]+.*" stdout)))) - ;;;----------------------------------------------------------- ;;; thin_restore scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..325d697 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,11 @@ +use std::path::{Path, PathBuf}; + +pub mod xml_generator; + +pub fn mk_path(dir: &Path, file: &str) -> PathBuf { + let mut p = PathBuf::new(); + p.push(dir); + p.push(PathBuf::from(file)); + p +} + diff --git a/tests/common/xml_generator.rs b/tests/common/xml_generator.rs new file mode 100644 index 0000000..eb388ab --- /dev/null +++ b/tests/common/xml_generator.rs @@ -0,0 +1,539 @@ +use anyhow::{anyhow, Result}; +use rand::prelude::*; +use std::collections::VecDeque; +use std::fs::OpenOptions; +use std::ops::Range; +use std::path::Path; +use thinp::thin::xml; + +//------------------------------------------ + +pub trait XmlGen { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; +} + +pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { + let xml_out = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + let mut w = xml::XmlWriter::new(xml_out); + + g.generate_xml(&mut w) +} + +fn common_sb(nr_blocks: u64) -> xml::Superblock { + xml::Superblock { + uuid: "".to_string(), + time: 0, + transaction: 0, + flags: None, + version: None, + data_block_size: 32, + nr_data_blocks: nr_blocks, + metadata_snap: None, + } +} + +//------------------------------------------ + +pub struct EmptyPoolS {} + +impl XmlGen for EmptyPoolS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.superblock_b(&common_sb(1024))?; + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ + +pub struct SingleThinS { + pub offset: u64, + pub len: u64, + pub old_nr_data_blocks: u64, + pub new_nr_data_blocks: u64, +} + +impl SingleThinS { + pub fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self { + SingleThinS { + offset, + len, + old_nr_data_blocks, + new_nr_data_blocks, + } + } +} + +impl XmlGen for SingleThinS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.superblock_b(&common_sb(self.old_nr_data_blocks))?; + v.device_b(&xml::Device { + dev_id: 0, + mapped_blocks: self.len, + transaction: 0, + creation_time: 0, + snap_time: 0, + })?; + v.map(&xml::Map { + thin_begin: 0, + data_begin: self.offset, + time: 0, + len: self.len, + })?; + v.device_e()?; + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ + +pub struct FragmentedS { + pub nr_thins: u32, + pub thin_size: u64, + pub old_nr_data_blocks: u64, + pub new_nr_data_blocks: u64, +} + +impl FragmentedS { + pub fn new(nr_thins: u32, thin_size: u64) -> Self { + let old_size = (nr_thins as u64) * thin_size; + FragmentedS { + nr_thins, + thin_size, + old_nr_data_blocks: (nr_thins as u64) * thin_size, + new_nr_data_blocks: old_size * 3 / 4, + } + } +} + +#[derive(Clone)] +struct ThinRun { + thin_id: u32, + thin_begin: u64, + len: u64, +} + +#[derive(Clone, Debug, Copy)] +struct MappedRun { + thin_id: u32, + thin_begin: u64, + data_begin: u64, + len: u64, +} + +fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range) -> Vec { + let mut runs = Vec::new(); + let mut b = 0u64; + while b < total_len { + let len = u64::min( + total_len - b, + thread_rng().gen_range(run_len.start, run_len.end), + ); + runs.push(ThinRun { + thin_id: thin_id, + thin_begin: b, + len, + }); + b += len; + } + runs +} + +impl XmlGen for FragmentedS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + // Allocate each thin fully, in runs between 1 and 16. + let mut runs = Vec::new(); + for thin in 0..self.nr_thins { + runs.append(&mut mk_runs(thin, self.thin_size, 1..17)); + } + + // Shuffle + runs.shuffle(&mut rand::thread_rng()); + + // map across the data + let mut maps = Vec::new(); + let mut b = 0; + for r in &runs { + maps.push(MappedRun { + thin_id: r.thin_id, + thin_begin: r.thin_begin, + data_begin: b, + len: r.len, + }); + b += r.len; + } + + // drop half the mappings, which leaves us free runs + let mut dropped = Vec::new(); + for i in 0..maps.len() { + if i % 2 == 0 { + dropped.push(maps[i].clone()); + } + } + + // Unshuffle. This isn't strictly necc. but makes the xml + // more readable. + use std::cmp::Ordering; + maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) { + Ordering::Equal => l.thin_begin.cmp(&r.thin_begin), + o => o, + }); + + // write the xml + v.superblock_b(&common_sb(self.old_nr_data_blocks))?; + for thin in 0..self.nr_thins { + v.device_b(&xml::Device { + dev_id: thin, + mapped_blocks: self.thin_size, + transaction: 0, + creation_time: 0, + snap_time: 0, + })?; + + for m in &dropped { + if m.thin_id != thin { + continue; + } + + v.map(&xml::Map { + thin_begin: m.thin_begin, + data_begin: m.data_begin, + time: 0, + len: m.len, + })?; + } + + v.device_e()?; + } + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ + +struct Allocator { + runs: VecDeque>, +} + +impl Allocator { + fn new_shuffled(total_len: u64, run_len: Range) -> Allocator { + let mut runs = Vec::new(); + + let mut b = 0u64; + while b < total_len { + let len = u64::min( + total_len - b, + thread_rng().gen_range(run_len.start, run_len.end), + ); + runs.push(b..(b + len)); + b += len; + } + + runs.shuffle(&mut thread_rng()); + let runs: VecDeque> = runs.iter().map(|r| r.clone()).collect(); + Allocator { runs } + } + + #[allow(dead_code)] + fn is_empty(&self) -> bool { + self.runs.is_empty() + } + + fn alloc(&mut self, len: u64) -> Result>> { + let mut len = len; + let mut runs = Vec::new(); + + while len > 0 { + let r = self.runs.pop_front(); + + if r.is_none() { + return Err(anyhow!("could not allocate; out of space")); + } + + let r = r.unwrap(); + let rlen = r.end - r.start; + if len < rlen { + runs.push(r.start..(r.start + len)); + + // We need to push something back. + self.runs.push_front((r.start + len)..r.end); + len = 0; + } else { + runs.push(r.start..r.end); + len -= rlen; + } + } + + Ok(runs) + } +} + +// Having explicitly unmapped regions makes it easier to +// apply snapshots. +#[derive(Clone)] +enum Run { + Mapped { data_begin: u64, len: u64 }, + UnMapped { len: u64 }, +} + +impl Run { + #[allow(dead_code)] + fn len(&self) -> u64 { + match self { + Run::Mapped { + data_begin: _data_begin, + len, + } => *len, + Run::UnMapped { len } => *len, + } + } + + fn split(&self, n: u64) -> (Option, Option) { + if n == 0 { + return (None, Some(self.clone())); + } else { + if self.len() <= n { + return (Some(self.clone()), None); + } else { + match self { + Run::Mapped { data_begin, len } => ( + Some(Run::Mapped { + data_begin: *data_begin, + len: n, + }), + Some(Run::Mapped { + data_begin: data_begin + n, + len: len - n, + }), + ), + Run::UnMapped { len } => ( + Some(Run::UnMapped { len: n }), + Some(Run::UnMapped { len: len - n }), + ), + } + } + } + } +} + +#[derive(Clone)] +struct ThinDev { + thin_id: u32, + dev_size: u64, + runs: Vec, +} + +impl ThinDev { + fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.device_b(&xml::Device { + dev_id: self.thin_id, + mapped_blocks: self.dev_size, + transaction: 0, + creation_time: 0, + snap_time: 0, + })?; + + let mut b = 0; + for r in &self.runs { + match r { + Run::Mapped { data_begin, len } => { + v.map(&xml::Map { + thin_begin: b, + data_begin: *data_begin, + time: 0, + len: *len, + })?; + b += len; + } + Run::UnMapped { len } => { + b += len; + } + } + } + + v.device_e()?; + Ok(()) + } +} + +#[derive(Clone)] +enum SnapRunType { + Same, + Diff, + Hole, +} + +#[derive(Clone)] +struct SnapRun(SnapRunType, u64); + +fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result { + let mut runs = Vec::new(); + let mut b = 0; + while b < total_len { + let len = u64::min(thread_rng().gen_range(16, 64), total_len - b); + match thread_rng().gen_range(0, 2) { + 0 => { + for data in allocator.alloc(len)? { + assert!(data.end >= data.start); + runs.push(Run::Mapped { + data_begin: data.start, + len: data.end - data.start, + }); + } + } + 1 => { + runs.push(Run::UnMapped { len }); + } + _ => { + return Err(anyhow!("bad value returned from rng")); + } + }; + + b += len; + } + + Ok(ThinDev { + thin_id, + dev_size: total_len, + runs, + }) +} + +fn mk_snap_mapping( + total_len: u64, + run_len: Range, + same_percent: usize, + diff_percent: usize, +) -> Vec { + let mut runs = Vec::new(); + + let mut b = 0u64; + while b < total_len { + let len = u64::min( + total_len - b, + thread_rng().gen_range(run_len.start, run_len.end), + ); + + let n = thread_rng().gen_range(0, 100); + + if n < same_percent { + runs.push(SnapRun(SnapRunType::Same, len)); + } else if n < diff_percent { + runs.push(SnapRun(SnapRunType::Diff, len)); + } else { + runs.push(SnapRun(SnapRunType::Hole, len)); + } + + b += len; + } + + runs +} + +fn split_runs(mut n: u64, runs: &Vec) -> (Vec, Vec) { + let mut before = Vec::new(); + let mut after = Vec::new(); + + for r in runs { + match r.split(n) { + (Some(lhs), None) => { + before.push(lhs); + } + (Some(lhs), Some(rhs)) => { + before.push(lhs); + after.push(rhs); + } + (None, Some(rhs)) => { + after.push(rhs); + } + (None, None) => {} + } + n -= r.len(); + } + + (before, after) +} + +fn apply_snap_runs( + origin: &Vec, + snap: &Vec, + allocator: &mut Allocator, +) -> Result> { + let mut origin = origin.clone(); + let mut runs = Vec::new(); + + for SnapRun(st, slen) in snap { + let (os, rest) = split_runs(*slen, &origin); + match st { + SnapRunType::Same => { + for o in os { + runs.push(o); + } + } + SnapRunType::Diff => { + for data in allocator.alloc(*slen)? { + runs.push(Run::Mapped { + data_begin: data.start, + len: data.end - data.start, + }); + } + } + SnapRunType::Hole => { + runs.push(Run::UnMapped { len: *slen }); + } + } + + origin = rest; + } + + Ok(runs) +} + +// Snapshots share mappings, not neccessarily the entire ranges. +pub struct SnapS { + pub len: u64, + pub nr_snaps: u32, + + // Snaps will differ from the origin by this percentage + pub percent_change: usize, + pub old_nr_data_blocks: u64, + pub new_nr_data_blocks: u64, +} + +impl SnapS { + pub fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self { + let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100; + let old_nr_data_blocks = len + 3 * delta; + let new_nr_data_blocks = len + 2 * delta; + + SnapS { + len, + nr_snaps, + percent_change, + old_nr_data_blocks, + new_nr_data_blocks, + } + } +} + +impl XmlGen for SnapS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512); + let origin = mk_origin(0, self.len, &mut allocator)?; + + v.superblock_b(&common_sb(self.old_nr_data_blocks))?; + origin.emit(v)?; + v.superblock_e()?; + + Ok(()) + } +} + +//------------------------------------------ diff --git a/tests/thin_check.rs b/tests/thin_check.rs new file mode 100644 index 0000000..385050d --- /dev/null +++ b/tests/thin_check.rs @@ -0,0 +1,173 @@ +use anyhow::Result; +use duct::{cmd, Expression}; +use std::path::{Path, PathBuf}; +use std::str::from_utf8; +use tempfile::{tempdir, TempDir}; +use thinp::file_utils; +use thinp::version::TOOLS_VERSION; + +mod common; + +use common::mk_path; +use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; + +//------------------------------------------ + +macro_rules! thin_check { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_check", args).stdout_capture().stderr_capture() + } + }; +} + +// Returns stderr, a non zero status must be returned +fn run_fail(command: Expression) -> Result { + let output = command.stderr_capture().unchecked().run()?; + assert!(!output.status.success()); + Ok(from_utf8(&output.stderr[0..]).unwrap().to_string()) +} + +fn mk_valid_md(dir: &TempDir) -> Result { + let xml = mk_path(dir.path(), "meta.xml"); + let md = mk_path(dir.path(), "meta.bin"); + + let mut gen = SingleThinS::new(0, 1024, 2048, 2048); + write_xml(&xml, &mut gen)?; + + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + cmd!("bin/thin_restore", "-i", xml, "-o", &md).run()?; + Ok(md) +} + +fn mk_corrupt_md(dir: &TempDir) -> Result { + let md = mk_path(dir.path(), "meta.bin"); + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + Ok(md) +} + +fn accepts_flag(flag: &str) -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + thin_check!(flag, &md).run()?; + Ok(()) +} + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_check!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_check!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &'static str = "Usage: thin_check [options] {device|file}\nOptions:\n {-q|--quiet}\n {-h|--help}\n {-V|--version}\n {-m|--metadata-snap}\n {--override-mapping-root}\n {--clear-needs-check-flag}\n {--ignore-non-fatal-errors}\n {--skip-mappings}\n {--super-block-only}"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_check!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_check!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn rejects_bad_option() -> Result<()> { + let stderr = run_fail(thin_check!("--hedgehogs-only"))?; + assert!(stderr.contains("unrecognized option \'--hedgehogs-only\'")); + Ok(()) +} + +#[test] +fn accepts_superblock_only() -> Result<()> { + accepts_flag("--super-block-only") +} + +#[test] +fn accepts_skip_mappings() -> Result<()> { + accepts_flag("--skip-mappings") +} + +#[test] +fn accepts_ignore_non_fatal_errors() -> Result<()> { + accepts_flag("--ignore-non-fatal-errors") +} + +#[test] +fn accepts_clear_needs_check_flag() -> Result<()> { + accepts_flag("--clear-needs-check-flag") +} + +#[test] +fn accepts_quiet() -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + + let output = thin_check!("--quiet", &md).run()?; + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +#[test] +fn detects_corrupt_superblock_with_superblock_only() -> Result<()> { + let dir = tempdir()?; + let md = mk_corrupt_md(&dir)?; + let output = thin_check!("--super-block-only", &md).unchecked().run()?; + assert!(!output.status.success()); + Ok(()) +} + +#[test] +fn prints_help_message_for_tiny_metadata() -> Result<()> { + let dir = tempdir()?; + let md = mk_path(dir.path(), "meta.bin"); + let _file = file_utils::create_sized_file(&md, 1024); + let stderr = run_fail(thin_check!(&md))?; + assert!(stderr.contains("Metadata device/file too small. Is this binary metadata?")); + Ok(()) +} + +#[test] +fn spot_xml_data() -> Result<()> { + let dir = tempdir()?; + let xml = mk_path(dir.path(), "meta.xml"); + + let mut gen = FragmentedS::new(4, 10240); + write_xml(&xml, &mut gen)?; + + let stderr = run_fail(thin_check!(&xml))?; + eprintln!("{}", stderr); + assert!( + stderr.contains("This looks like XML. thin_check only checks the binary metadata format.") + ); + Ok(()) +} + +#[test] +fn prints_info_fields() -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + let stdout = thin_check!(&md).read()?; + assert!(stdout.contains("TRANSACTION_ID=")); + assert!(stdout.contains("METADATA_FREE_BLOCKS=")); + Ok(()) +} + +//------------------------------------------ diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index 0291340..5686525 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -1,16 +1,20 @@ use anyhow::{anyhow, Result}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use rand::prelude::*; -use std::collections::VecDeque; use std::fs::OpenOptions; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; -use std::ops::Range; use std::path::{Path, PathBuf}; use tempfile::tempdir; use thinp::file_utils; use thinp::thin::xml::{self, Visit}; +mod common; +use common::mk_path; +use common::xml_generator::{ + write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen, +}; + //------------------------------------ #[derive(Debug)] @@ -251,25 +255,6 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> { //------------------------------------ -fn mk_path(dir: &Path, file: &str) -> PathBuf { - let mut p = PathBuf::new(); - p.push(dir); - p.push(PathBuf::from(file)); - p -} - -fn generate_xml(path: &Path, g: &mut dyn Scenario) -> Result<()> { - let xml_out = OpenOptions::new() - .read(false) - .write(true) - .create(true) - .truncate(true) - .open(path)?; - let mut w = xml::XmlWriter::new(xml_out); - - g.generate_xml(&mut w) -} - fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> { let input = OpenOptions::new().read(true).write(false).open(xml_path)?; @@ -304,17 +289,19 @@ fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> { } trait Scenario { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; fn get_new_nr_blocks(&self) -> u64; } -fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> { +fn test_shrink(scenario: &mut S) -> Result<()> +where + S: Scenario + XmlGen, +{ let dir = tempdir()?; let xml_before = mk_path(dir.path(), "before.xml"); let xml_after = mk_path(dir.path(), "after.xml"); let data_path = mk_path(dir.path(), "metadata.bin"); - generate_xml(&xml_before, scenario)?; + write_xml(&xml_before, scenario)?; create_data_file(&data_path, &xml_before)?; let mut rng = rand::thread_rng(); @@ -332,28 +319,7 @@ fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> { //------------------------------------ -fn common_sb(nr_blocks: u64) -> xml::Superblock { - xml::Superblock { - uuid: "".to_string(), - time: 0, - transaction: 0, - flags: None, - version: None, - data_block_size: 32, - nr_data_blocks: nr_blocks, - metadata_snap: None, - } -} - -struct EmptyPoolS {} - impl Scenario for EmptyPoolS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.superblock_b(&common_sb(1024))?; - v.superblock_e()?; - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { 512 } @@ -367,45 +333,7 @@ fn shrink_empty_pool() -> Result<()> { //------------------------------------ -struct SingleThinS { - offset: u64, - len: u64, - old_nr_data_blocks: u64, - new_nr_data_blocks: u64, -} - -impl SingleThinS { - fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self { - SingleThinS { - offset, - len, - old_nr_data_blocks, - new_nr_data_blocks, - } - } -} - impl Scenario for SingleThinS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - v.device_b(&xml::Device { - dev_id: 0, - mapped_blocks: self.len, - transaction: 0, - creation_time: 0, - snap_time: 0, - })?; - v.map(&xml::Map { - thin_begin: 0, - data_begin: self.offset, - time: 0, - len: self.len, - })?; - v.device_e()?; - v.superblock_e()?; - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { self.new_nr_data_blocks } @@ -452,128 +380,7 @@ fn shrink_insufficient_space() -> Result<()> { //------------------------------------ -struct FragmentedS { - nr_thins: u32, - thin_size: u64, - old_nr_data_blocks: u64, - new_nr_data_blocks: u64, -} - -impl FragmentedS { - fn new(nr_thins: u32, thin_size: u64) -> Self { - let old_size = (nr_thins as u64) * thin_size; - FragmentedS { - nr_thins, - thin_size, - old_nr_data_blocks: (nr_thins as u64) * thin_size, - new_nr_data_blocks: old_size * 3 / 4, - } - } -} - -#[derive(Clone)] -struct ThinRun { - thin_id: u32, - thin_begin: u64, - len: u64, -} - -#[derive(Clone, Debug, Copy)] -struct MappedRun { - thin_id: u32, - thin_begin: u64, - data_begin: u64, - len: u64, -} - -fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range) -> Vec { - let mut runs = Vec::new(); - let mut b = 0u64; - while b < total_len { - let len = u64::min( - total_len - b, - thread_rng().gen_range(run_len.start, run_len.end), - ); - runs.push(ThinRun { - thin_id: thin_id, - thin_begin: b, - len, - }); - b += len; - } - runs -} - impl Scenario for FragmentedS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - // Allocate each thin fully, in runs between 1 and 16. - let mut runs = Vec::new(); - for thin in 0..self.nr_thins { - runs.append(&mut mk_runs(thin, self.thin_size, 1..17)); - } - - // Shuffle - runs.shuffle(&mut rand::thread_rng()); - - // map across the data - let mut maps = Vec::new(); - let mut b = 0; - for r in &runs { - maps.push(MappedRun { - thin_id: r.thin_id, - thin_begin: r.thin_begin, - data_begin: b, - len: r.len, - }); - b += r.len; - } - - // drop half the mappings, which leaves us free runs - let mut dropped = Vec::new(); - for i in 0..maps.len() { - if i % 2 == 0 { - dropped.push(maps[i].clone()); - } - } - - // Unshuffle. This isn't strictly necc. but makes the xml - // more readable. - use std::cmp::Ordering; - maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) { - Ordering::Equal => l.thin_begin.cmp(&r.thin_begin), - o => o, - }); - - // write the xml - v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - for thin in 0..self.nr_thins { - v.device_b(&xml::Device { - dev_id: thin, - mapped_blocks: self.thin_size, - transaction: 0, - creation_time: 0, - snap_time: 0, - })?; - - for m in &dropped { - if m.thin_id != thin { - continue; - } - - v.map(&xml::Map { - thin_begin: m.thin_begin, - data_begin: m.data_begin, - time: 0, - len: m.len, - })?; - } - - v.device_e()?; - } - v.superblock_e()?; - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { self.new_nr_data_blocks } @@ -605,321 +412,7 @@ fn shrink_fragmented_thin_64() -> Result<()> { //------------------------------------ -struct Allocator { - runs: VecDeque>, -} - -impl Allocator { - fn new_shuffled(total_len: u64, run_len: Range) -> Allocator { - let mut runs = Vec::new(); - - let mut b = 0u64; - while b < total_len { - let len = u64::min( - total_len - b, - thread_rng().gen_range(run_len.start, run_len.end), - ); - runs.push(b..(b + len)); - b += len; - } - - runs.shuffle(&mut thread_rng()); - let runs: VecDeque> = runs.iter().map(|r| r.clone()).collect(); - Allocator { runs } - } - - fn is_empty(&self) -> bool { - self.runs.is_empty() - } - - fn alloc(&mut self, len: u64) -> Result>> { - let mut len = len; - let mut runs = Vec::new(); - - while len > 0 { - let r = self.runs.pop_front(); - - if r.is_none() { - return Err(anyhow!("could not allocate; out of space")); - } - - let mut r = r.unwrap(); - let rlen = r.end - r.start; - if len < rlen { - runs.push(r.start..(r.start + len)); - - // We need to push something back. - self.runs.push_front((r.start + len)..r.end); - len = 0; - } else { - runs.push(r.start..r.end); - len -= rlen; - } - } - - Ok(runs) - } -} - -// Having explicitly unmapped regions makes it easier to -// apply snapshots. -#[derive(Clone)] -enum Run { - Mapped { data_begin: u64, len: u64 }, - UnMapped { len: u64 }, -} - -impl Run { - fn len(&self) -> u64 { - match self { - Run::Mapped { - data_begin: _data_begin, - len, - } => *len, - Run::UnMapped { len } => *len, - } - } - - fn split(&self, n: u64) -> (Option, Option) { - if n == 0 { - return (None, Some(self.clone())); - } else { - if self.len() <= n { - return (Some(self.clone()), None); - } else { - match self { - Run::Mapped { data_begin, len } => ( - Some(Run::Mapped { - data_begin: *data_begin, - len: n, - }), - Some(Run::Mapped { - data_begin: data_begin + n, - len: len - n, - }), - ), - Run::UnMapped { len } => ( - Some(Run::UnMapped { len: n }), - Some(Run::UnMapped { len: len - n }), - ), - } - } - } - } -} - -#[derive(Clone)] -struct ThinDev { - thin_id: u32, - dev_size: u64, - runs: Vec, -} - -impl ThinDev { - fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.device_b(&xml::Device { - dev_id: self.thin_id, - mapped_blocks: self.dev_size, - transaction: 0, - creation_time: 0, - snap_time: 0, - })?; - - let mut b = 0; - for r in &self.runs { - match r { - Run::Mapped { data_begin, len } => { - v.map(&xml::Map { - thin_begin: b, - data_begin: *data_begin, - time: 0, - len: *len, - })?; - b += len; - } - Run::UnMapped { len } => { - b += len; - } - } - } - - v.device_e()?; - Ok(()) - } -} - -#[derive(Clone)] -enum SnapRunType { - Same, - Diff, - Hole, -} - -#[derive(Clone)] -struct SnapRun(SnapRunType, u64); - -fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result { - let mut runs = Vec::new(); - let mut b = 0; - while b < total_len { - let len = u64::min(thread_rng().gen_range(16, 64), total_len - b); - match thread_rng().gen_range(0, 2) { - 0 => { - for data in allocator.alloc(len)? { - assert!(data.end >= data.start); - runs.push(Run::Mapped { - data_begin: data.start, - len: data.end - data.start, - }); - } - } - 1 => { - runs.push(Run::UnMapped { len }); - } - _ => { - return Err(anyhow!("bad value returned from rng")); - } - }; - - b += len; - } - - Ok(ThinDev { - thin_id, - dev_size: total_len, - runs, - }) -} - -fn mk_snap_mapping( - total_len: u64, - run_len: Range, - same_percent: usize, - diff_percent: usize, -) -> Vec { - let mut runs = Vec::new(); - - let mut b = 0u64; - while b < total_len { - let len = u64::min( - total_len - b, - thread_rng().gen_range(run_len.start, run_len.end), - ); - - let n = thread_rng().gen_range(0, 100); - - if n < same_percent { - runs.push(SnapRun(SnapRunType::Same, len)); - } else if n < diff_percent { - runs.push(SnapRun(SnapRunType::Diff, len)); - } else { - runs.push(SnapRun(SnapRunType::Hole, len)); - } - - b += len; - } - - runs -} - -fn split_runs(mut n: u64, runs: &Vec) -> (Vec, Vec) { - let mut before = Vec::new(); - let mut after = Vec::new(); - - for r in runs { - match r.split(n) { - (Some(lhs), None) => { - before.push(lhs); - } - (Some(lhs), Some(rhs)) => { - before.push(lhs); - after.push(rhs); - } - (None, Some(rhs)) => { - after.push(rhs); - } - (None, None) => {} - } - n -= r.len(); - } - - (before, after) -} - -fn apply_snap_runs( - origin: &Vec, - snap: &Vec, - allocator: &mut Allocator, -) -> Result> { - let mut origin = origin.clone(); - let mut runs = Vec::new(); - - for SnapRun(st, slen) in snap { - let (os, rest) = split_runs(*slen, &origin); - match st { - SnapRunType::Same => { - for o in os { - runs.push(o); - } - } - SnapRunType::Diff => { - for data in allocator.alloc(*slen)? { - runs.push(Run::Mapped { - data_begin: data.start, - len: data.end - data.start, - }); - } - } - SnapRunType::Hole => { - runs.push(Run::UnMapped { len: *slen }); - } - } - - origin = rest; - } - - Ok(runs) -} - -// Snapshots share mappings, not neccessarily the entire ranges. -struct SnapS { - len: u64, - nr_snaps: u32, - - // Snaps will differ from the origin by this percentage - percent_change: usize, - old_nr_data_blocks: u64, - new_nr_data_blocks: u64, -} - -impl SnapS { - fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self { - let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100; - let old_nr_data_blocks = len + 3 * delta; - let new_nr_data_blocks = len + 2 * delta; - - SnapS { - len, - nr_snaps, - percent_change, - old_nr_data_blocks, - new_nr_data_blocks, - } - } -} - impl Scenario for SnapS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512); - let origin = mk_origin(0, self.len, &mut allocator)?; - - v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - origin.emit(v)?; - v.superblock_e()?; - - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { self.new_nr_data_blocks }