diff --git a/src/bin/cache_restore.rs b/src/bin/cache_restore.rs new file mode 100644 index 0000000..9375c91 --- /dev/null +++ b/src/bin/cache_restore.rs @@ -0,0 +1,77 @@ +extern crate clap; +extern crate thinp; + +use atty::Stream; +use clap::{App, Arg}; +use std::path::Path; +use std::process; +use std::process::exit; +use std::sync::Arc; +use thinp::cache::restore::{restore, CacheRestoreOptions}; +use thinp::file_utils; +use thinp::report::*; + +fn main() { + let parser = App::new("cache_restore") + .version(thinp::version::tools_version()) + .about("Convert XML format metadata to binary.") + .arg( + Arg::with_name("OVERRIDE_MAPPING_ROOT") + .help("Specify a mapping root to use") + .long("override-mapping-root") + .value_name("OVERRIDE_MAPPING_ROOT") + .takes_value(true), + ) + .arg( + Arg::with_name("INPUT") + .help("Specify the input xml") + .short("i") + .long("input") + .value_name("INPUT") + .required(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify the output device to check") + .short("o") + .long("output") + .value_name("OUTPUT") + .required(true), + ) + .arg( + Arg::with_name("SYNC_IO") + .help("Force use of synchronous io") + .long("sync-io"), + ); + + let matches = parser.get_matches(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); + + if !file_utils::file_exists(input_file) { + eprintln!("Couldn't find input file '{:?}'.", &input_file); + exit(1); + } + + let report; + + if matches.is_present("QUIET") { + report = std::sync::Arc::new(mk_quiet_report()); + } else if atty::is(Stream::Stdout) { + report = std::sync::Arc::new(mk_progress_bar_report()); + } else { + report = Arc::new(mk_simple_report()); + } + + let opts = CacheRestoreOptions { + input: &input_file, + output: &output_file, + async_io: !matches.is_present("SYNC_IO"), + report, + }; + + if let Err(reason) = restore(opts) { + println!("{}", reason); + process::exit(1); + } +} diff --git a/src/cache/check.rs b/src/cache/check.rs index 5428735..82e37c1 100644 --- a/src/cache/check.rs +++ b/src/cache/check.rs @@ -12,7 +12,7 @@ use crate::pdata::array_walker::*; use crate::pdata::bitset::*; use crate::pdata::space_map::*; use crate::pdata::space_map_checker::*; -use crate::pdata::space_map_disk::*; +use crate::pdata::space_map_common::*; use crate::pdata::unpack::unpack; use crate::report::*; @@ -274,8 +274,11 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { return Ok(()); } + // The discard bitset is optional and could be updated during device suspension. + // A restored metadata therefore comes with a zero-sized discard bitset, + // and also zeroed discard_block_size and discard_nr_blocks. let nr_origin_blocks; - if sb.flags.clean_shutdown { + if sb.flags.clean_shutdown && sb.discard_block_size > 0 && sb.discard_nr_blocks > 0 { let origin_sectors = sb.discard_block_size * sb.discard_nr_blocks; nr_origin_blocks = Some(origin_sectors / sb.data_block_size as u64); } else { diff --git a/src/cache/dump.rs b/src/cache/dump.rs index 22f16fd..f80421a 100644 --- a/src/cache/dump.rs +++ b/src/cache/dump.rs @@ -10,6 +10,7 @@ use crate::cache::xml::{self, MetadataVisitor}; use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; use crate::pdata::array::{self, ArrayBlock}; use crate::pdata::array_walker::*; +use crate::pdata::bitset::{read_bitset, CheckedBitSet}; //------------------------------------------ @@ -78,49 +79,12 @@ mod format1 { mod format2 { use super::*; - //------------------- - // Dirty bitset visitor - pub struct DirtyVisitor { - nr_entries: usize, - bits: Mutex, - } - - impl DirtyVisitor { - pub fn new(nr_entries: usize) -> Self { - DirtyVisitor { - nr_entries, // number of bits - bits: Mutex::new(FixedBitSet::with_capacity(nr_entries)), - } - } - - pub fn get_bits(self) -> FixedBitSet { - self.bits.into_inner().unwrap() - } - } - - impl ArrayVisitor for DirtyVisitor { - fn visit(&self, index: u64, b: ArrayBlock) -> array::Result<()> { - let mut pos = (index as usize * (b.header.max_entries as usize)) << 6; - for bits in b.values.iter() { - for bi in 0..64u64 { - if pos >= self.nr_entries { - break; - } - - self.bits.lock().unwrap().set(pos, bits & (1 << bi) != 0); - pos += 1; - } - } - Ok(()) - } - } - //------------------- // Mapping visitor struct Inner<'a> { visitor: &'a mut dyn MetadataVisitor, - dirty_bits: FixedBitSet, + dirty_bits: CheckedBitSet, valid_mappings: FixedBitSet, } @@ -131,7 +95,7 @@ mod format2 { impl<'a> MappingEmitter<'a> { pub fn new( nr_entries: usize, - dirty_bits: FixedBitSet, + dirty_bits: CheckedBitSet, visitor: &'a mut dyn MetadataVisitor, ) -> MappingEmitter<'a> { MappingEmitter { @@ -159,7 +123,13 @@ mod format2 { } let mut inner = self.inner.lock().unwrap(); - let dirty = inner.dirty_bits.contains(cblock as usize); + let dirty; + if let Some(bit) = inner.dirty_bits.contains(cblock as usize) { + dirty = bit; + } else { + // default to dirty if the bitset is damaged + dirty = true; + } let m = xml::Map { cblock, oblock: map.oblock, @@ -266,17 +236,20 @@ fn dump_metadata(ctx: &Context, sb: &Superblock, _repair: bool) -> anyhow::Resul } 2 => { // We need to walk the dirty bitset first. - let w = ArrayWalker::new(engine.clone(), false); - let mut v = format2::DirtyVisitor::new(sb.cache_blocks as usize); - + let dirty_bits; if let Some(root) = sb.dirty_root { - w.walk(&mut v, root)?; + let (bits, errs) = + read_bitset(engine.clone(), root, sb.cache_blocks as usize, false); + // TODO: allow errors in repair mode + if errs.is_some() { + return Err(anyhow!("errors in bitset {}", errs.unwrap())); + } + dirty_bits = bits; } else { // FIXME: is there a way this can legally happen? eg, // a crash of a freshly created cache? return Err(anyhow!("format 2 selected, but no dirty bitset present")); } - let dirty_bits = v.get_bits(); let w = ArrayWalker::new(engine.clone(), false); let mut emitter = diff --git a/src/cache/hint.rs b/src/cache/hint.rs index b066645..7dcc709 100644 --- a/src/cache/hint.rs +++ b/src/cache/hint.rs @@ -1,3 +1,5 @@ +use anyhow::Result; +use byteorder::WriteBytesExt; use nom::IResult; use std::convert::TryInto; @@ -26,4 +28,19 @@ impl Unpack for Hint { } } +impl Pack for Hint { + fn pack(&self, data: &mut W) -> Result<()> { + for v in &self.hint { + data.write_u8(*v)?; + } + Ok(()) + } +} + +impl Default for Hint { + fn default() -> Self { + Hint { hint: [0; 4] } + } +} + //------------------------------------------ diff --git a/src/cache/mapping.rs b/src/cache/mapping.rs index ac5a036..eed1840 100644 --- a/src/cache/mapping.rs +++ b/src/cache/mapping.rs @@ -1,3 +1,5 @@ +use anyhow::Result; +use byteorder::WriteBytesExt; use nom::number::complete::*; use nom::IResult; @@ -51,4 +53,20 @@ impl Unpack for Mapping { } } +impl Pack for Mapping { + fn pack(&self, data: &mut W) -> Result<()> { + let m: u64 = (self.oblock << 16) | self.flags as u64; + m.pack(data) + } +} + +impl Default for Mapping { + fn default() -> Self { + Mapping { + oblock: 0, + flags: 0, + } + } +} + //------------------------------------------ diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 5aa61fa..ceb8867 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -2,5 +2,6 @@ pub mod check; pub mod dump; pub mod hint; pub mod mapping; +pub mod restore; pub mod superblock; pub mod xml; diff --git a/src/cache/restore.rs b/src/cache/restore.rs new file mode 100644 index 0000000..d4a972b --- /dev/null +++ b/src/cache/restore.rs @@ -0,0 +1,293 @@ +use anyhow::{anyhow, Result}; + +use std::convert::TryInto; +use std::fs::OpenOptions; +use std::io::Cursor; +use std::ops::Deref; +use std::path::Path; +use std::sync::Arc; + +use crate::cache::hint::Hint; +use crate::cache::mapping::{Mapping, MappingFlags}; +use crate::cache::superblock::*; +use crate::cache::xml::{self, MetadataVisitor, Visit}; +use crate::io_engine::*; +use crate::math::*; +use crate::pdata::array_builder::*; +use crate::pdata::space_map::*; +use crate::pdata::space_map_metadata::*; +use crate::pdata::unpack::Pack; +use crate::report::*; +use crate::write_batcher::*; + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + +//------------------------------------------ + +pub struct CacheRestoreOptions<'a> { + pub input: &'a Path, + pub output: &'a Path, + pub async_io: bool, + pub report: Arc, +} + +struct Context { + _report: Arc, + engine: Arc, +} + +fn mk_context(opts: &CacheRestoreOptions) -> anyhow::Result { + let engine: Arc; + + if opts.async_io { + engine = Arc::new(AsyncIoEngine::new(opts.output, MAX_CONCURRENT_IO, true)?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine = Arc::new(SyncIoEngine::new(opts.output, nr_threads, true)?); + } + + Ok(Context { + _report: opts.report.clone(), + engine, + }) +} + +//------------------------------------------ + +struct RestoreResult { + sb: xml::Superblock, + mapping_root: u64, + dirty_root: Option, + hint_root: u64, + discard_root: u64, +} + +struct Restorer<'a> { + write_batcher: &'a mut WriteBatcher, + sb: Option, + mapping_builder: Option>, + dirty_builder: Option>, + hint_builder: Option>, + mapping_root: Option, + dirty_root: Option, + hint_root: Option, + discard_root: Option, + dirty_bits: (u32, u64), +} + +impl<'a> Restorer<'a> { + fn new(w: &'a mut WriteBatcher) -> Restorer<'a> { + Restorer { + write_batcher: w, + sb: None, + mapping_builder: None, + dirty_builder: None, + hint_builder: None, + mapping_root: None, + dirty_root: None, + hint_root: None, + discard_root: None, + dirty_bits: (0, 0), + } + } + + fn get_result(self) -> Result { + self.write_batcher.flush()?; + + if self.sb.is_none() || self.discard_root.is_none() { + return Err(anyhow!("No superblock found in xml file")); + } + if self.mapping_root.is_none() || self.hint_root.is_none() { + return Err(anyhow!("No mappings or hints sections in xml file")); + } + Ok(RestoreResult { + sb: self.sb.unwrap(), + mapping_root: self.mapping_root.unwrap(), + dirty_root: self.dirty_root, + hint_root: self.hint_root.unwrap(), + discard_root: self.discard_root.unwrap(), + }) + } +} + +impl<'a> MetadataVisitor for Restorer<'a> { + fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { + self.sb = Some(sb.clone()); + self.write_batcher.alloc()?; + self.mapping_builder = Some(ArrayBuilder::new(sb.nr_cache_blocks as u64)); + self.dirty_builder = Some(ArrayBuilder::new(div_up(sb.nr_cache_blocks as u64, 64))); + self.hint_builder = Some(ArrayBuilder::new(sb.nr_cache_blocks as u64)); + + let discard_builder = ArrayBuilder::::new(0); // discard bitset is optional + self.discard_root = Some(discard_builder.complete(self.write_batcher)?); + + Ok(Visit::Continue) + } + + fn superblock_e(&mut self) -> Result { + Ok(Visit::Continue) + } + + fn mappings_b(&mut self) -> Result { + Ok(Visit::Continue) + } + + fn mappings_e(&mut self) -> Result { + let mut mapping_builder = None; + std::mem::swap(&mut self.mapping_builder, &mut mapping_builder); + if let Some(builder) = mapping_builder { + self.mapping_root = Some(builder.complete(self.write_batcher)?); + } + + // push the bufferred trailing bits + let b = self.dirty_builder.as_mut().unwrap(); + b.push_value( + self.write_batcher, + self.dirty_bits.0 as u64, + self.dirty_bits.1, + )?; + + let mut dirty_builder = None; + std::mem::swap(&mut self.dirty_builder, &mut dirty_builder); + if let Some(builder) = dirty_builder { + self.dirty_root = Some(builder.complete(self.write_batcher)?); + } + + Ok(Visit::Continue) + } + + fn mapping(&mut self, m: &xml::Map) -> Result { + let map = Mapping { + oblock: m.oblock, + flags: MappingFlags::Valid as u32, + }; + let mapping_builder = self.mapping_builder.as_mut().unwrap(); + mapping_builder.push_value(self.write_batcher, m.cblock as u64, map)?; + + if m.dirty { + let index = m.cblock >> 6; + let bi = m.cblock & 63; + if index == self.dirty_bits.0 { + self.dirty_bits.1 |= 1 << bi; + } else { + let dirty_builder = self.dirty_builder.as_mut().unwrap(); + dirty_builder.push_value( + self.write_batcher, + self.dirty_bits.0 as u64, + self.dirty_bits.1, + )?; + self.dirty_bits.0 = index; + self.dirty_bits.1 = 0; + } + } + + Ok(Visit::Continue) + } + + fn hints_b(&mut self) -> Result { + Ok(Visit::Continue) + } + + fn hints_e(&mut self) -> Result { + let mut hint_builder = None; + std::mem::swap(&mut self.hint_builder, &mut hint_builder); + if let Some(builder) = hint_builder { + self.hint_root = Some(builder.complete(self.write_batcher)?); + } + Ok(Visit::Continue) + } + + fn hint(&mut self, h: &xml::Hint) -> Result { + let hint = Hint { + hint: h.data[..].try_into().unwrap(), + }; + let hint_builder = self.hint_builder.as_mut().unwrap(); + hint_builder.push_value(self.write_batcher, h.cblock as u64, hint)?; + Ok(Visit::Continue) + } + + fn discards_b(&mut self) -> Result { + Ok(Visit::Continue) + } + + fn discards_e(&mut self) -> Result { + Ok(Visit::Continue) + } + + fn discard(&mut self, _d: &xml::Discard) -> Result { + Ok(Visit::Continue) + } + + fn eof(&mut self) -> Result { + Ok(Visit::Continue) + } +} + +//------------------------------------------ + +fn build_metadata_sm(w: &mut WriteBatcher) -> Result> { + let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; + let mut cur = Cursor::new(&mut sm_root); + let sm_without_meta = clone_space_map(w.sm.lock().unwrap().deref())?; + let r = write_metadata_sm(w, sm_without_meta.deref())?; + r.pack(&mut cur)?; + + Ok(sm_root) +} + +//------------------------------------------ + +pub fn restore(opts: CacheRestoreOptions) -> Result<()> { + let input = OpenOptions::new() + .read(true) + .write(false) + .open(opts.input)?; + + let ctx = mk_context(&opts)?; + + let sm = core_sm(ctx.engine.get_nr_blocks(), u32::MAX); + let mut w = WriteBatcher::new(ctx.engine.clone(), sm.clone(), ctx.engine.get_batch_size()); + + // build cache mappings + let mut restorer = Restorer::new(&mut w); + xml::read(input, &mut restorer)?; + let result = restorer.get_result()?; + + // build metadata space map + let metadata_sm_root = build_metadata_sm(&mut w)?; + + let sb = Superblock { + flags: SuperblockFlags { + clean_shutdown: true, + needs_check: false, + }, + block: SUPERBLOCK_LOCATION, + version: 2, + policy_name: result.sb.policy.as_bytes().to_vec(), + policy_version: vec![2, 0, 0], + policy_hint_size: result.sb.hint_width, + metadata_sm_root, + mapping_root: result.mapping_root, + dirty_root: result.dirty_root, + hint_root: result.hint_root, + discard_root: result.discard_root, + discard_block_size: 0, + discard_nr_blocks: 0, + data_block_size: result.sb.block_size, + cache_blocks: result.sb.nr_cache_blocks, + compat_flags: 0, + compat_ro_flags: 0, + incompat_flags: 0, + read_hits: 0, + read_misses: 9, + write_hits: 0, + write_misses: 0, + }; + write_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?; + + Ok(()) +} + +//------------------------------------------ diff --git a/src/cache/superblock.rs b/src/cache/superblock.rs index 227fed3..23bc474 100644 --- a/src/cache/superblock.rs +++ b/src/cache/superblock.rs @@ -1,14 +1,19 @@ use anyhow::{anyhow, Result}; +use byteorder::{LittleEndian, WriteBytesExt}; use nom::{bytes::complete::*, number::complete::*, IResult}; +use std::io::Cursor; +use crate::checksum::*; use crate::io_engine::*; //------------------------------------------ +pub const SPACE_MAP_ROOT_SIZE: usize = 128; pub const SUPERBLOCK_LOCATION: u64 = 0; +const MAGIC: u64 = 0o6142003; // 0x18c403 in hex const POLICY_NAME_SIZE: usize = 16; -const SPACE_MAP_ROOT_SIZE: usize = 128; +const UUID_SIZE: usize = 16; //------------------------------------------ @@ -38,7 +43,6 @@ pub struct Superblock { pub discard_nr_blocks: u64, pub data_block_size: u32, - pub metadata_block_size: u32, pub cache_blocks: u32, pub compat_flags: u32, @@ -71,7 +75,7 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { let (i, discard_nr_blocks) = le_u64(i)?; let (i, data_block_size) = le_u32(i)?; - let (i, metadata_block_size) = le_u32(i)?; + let (i, _metadata_block_size) = le_u32(i)?; let (i, cache_blocks) = le_u32(i)?; let (i, compat_flags) = le_u32(i)?; @@ -115,7 +119,6 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { discard_block_size, discard_nr_blocks, data_block_size, - metadata_block_size, cache_blocks, compat_flags, compat_ro_flags, @@ -139,3 +142,79 @@ pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result { } //------------------------------------------ + +fn pack_superblock(sb: &Superblock, w: &mut W) -> Result<()> { + // checksum, which we don't know yet + w.write_u32::(0)?; + + // flags + let mut flags: u32 = 0; + if sb.flags.clean_shutdown { + flags |= 0x1; + } + if sb.flags.needs_check { + flags |= 0x2; + } + w.write_u32::(flags)?; + + w.write_u64::(sb.block)?; + w.write_all(&[0; UUID_SIZE])?; + w.write_u64::(MAGIC)?; + w.write_u32::(sb.version)?; + + let mut policy_name = [0u8; POLICY_NAME_SIZE]; + policy_name[..sb.policy_name.len()].copy_from_slice(&sb.policy_name[..]); + w.write_all(&policy_name)?; + + w.write_u32::(sb.policy_hint_size)?; + w.write_all(&sb.metadata_sm_root)?; + w.write_u64::(sb.mapping_root)?; + w.write_u64::(sb.hint_root)?; + + w.write_u64::(sb.discard_root)?; + w.write_u64::(sb.discard_block_size)?; + w.write_u64::(sb.discard_nr_blocks)?; + + w.write_u32::(sb.data_block_size)?; + // metadata block size + w.write_u32::((BLOCK_SIZE >> SECTOR_SHIFT) as u32)?; + w.write_u32::(sb.cache_blocks)?; + + w.write_u32::(sb.compat_flags)?; + w.write_u32::(sb.compat_ro_flags)?; + w.write_u32::(sb.incompat_flags)?; + + w.write_u32::(sb.read_hits)?; + w.write_u32::(sb.read_misses)?; + w.write_u32::(sb.write_hits)?; + w.write_u32::(sb.write_misses)?; + + w.write_u32::(sb.policy_version[0])?; + w.write_u32::(sb.policy_version[1])?; + w.write_u32::(sb.policy_version[2])?; + + if sb.dirty_root.is_some() { + w.write_u64::(sb.dirty_root.unwrap())?; + } + + Ok(()) +} + +pub fn write_superblock(engine: &dyn IoEngine, _loc: u64, sb: &Superblock) -> Result<()> { + let b = Block::zeroed(SUPERBLOCK_LOCATION); + + // pack the superblock + { + let mut cursor = Cursor::new(b.get_data()); + pack_superblock(sb, &mut cursor)?; + } + + // calculate the checksum + write_checksum(b.get_data(), BT::CACHE_SUPERBLOCK)?; + + // write + engine.write(&b)?; + Ok(()) +} + +//------------------------------------------ diff --git a/src/cache/xml.rs b/src/cache/xml.rs index cf5fa86..6a05744 100644 --- a/src/cache/xml.rs +++ b/src/cache/xml.rs @@ -1,10 +1,12 @@ -use anyhow::Result; -use base64::encode; -use std::{borrow::Cow, fmt::Display, io::Write}; +use anyhow::{anyhow, Result}; +use base64::{decode, encode}; +use std::io::{BufRead, BufReader}; +use std::io::{Read, Write}; -use quick_xml::events::attributes::Attribute; use quick_xml::events::{BytesEnd, BytesStart, Event}; -use quick_xml::Writer; +use quick_xml::{Reader, Writer}; + +use crate::xml::*; //--------------------------------------- @@ -73,18 +75,6 @@ impl XmlWriter { } } -fn mk_attr_<'a, T: Display>(n: T) -> Cow<'a, [u8]> { - let str = format!("{}", n); - Cow::Owned(str.into_bytes()) -} - -fn mk_attr(key: &[u8], value: T) -> Attribute { - Attribute { - key, - value: mk_attr_(value), - } -} - impl MetadataVisitor for XmlWriter { fn superblock_b(&mut self, sb: &Superblock) -> Result { let tag = b"superblock"; @@ -176,3 +166,129 @@ impl MetadataVisitor for XmlWriter { Ok(Visit::Continue) } } + +//------------------------------------------ + +fn parse_superblock(e: &BytesStart) -> Result { + let mut uuid: Option = None; + let mut block_size: Option = None; + let mut nr_cache_blocks: Option = None; + let mut policy: Option = None; + let mut hint_width: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"uuid" => uuid = Some(string_val(&kv)), + b"block_size" => block_size = Some(u32_val(&kv)?), + b"nr_cache_blocks" => nr_cache_blocks = Some(u32_val(&kv)?), + b"policy" => policy = Some(string_val(&kv)), + b"hint_width" => hint_width = Some(u32_val(&kv)?), + _ => return bad_attr("superblock", kv.key), + } + } + + let tag = "cache"; + + Ok(Superblock { + uuid: check_attr(tag, "uuid", uuid)?, + block_size: check_attr(tag, "block_size", block_size)?, + nr_cache_blocks: check_attr(tag, "nr_cache_blocks", nr_cache_blocks)?, + policy: check_attr(tag, "policy", policy)?, + hint_width: check_attr(tag, "hint_width", hint_width)?, + }) +} + +fn parse_mapping(e: &BytesStart) -> Result { + let mut cblock: Option = None; + let mut oblock: Option = None; + let mut dirty: Option = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"cache_block" => cblock = Some(u32_val(&kv)?), + b"origin_block" => oblock = Some(u64_val(&kv)?), + b"dirty" => dirty = Some(bool_val(&kv)?), + _ => return bad_attr("mapping", kv.key), + } + } + + let tag = "mapping"; + + Ok(Map { + cblock: check_attr(tag, "cache_block", cblock)?, + oblock: check_attr(tag, "origin_block", oblock)?, + dirty: check_attr(tag, "dirty", dirty)?, + }) +} + +fn parse_hint(e: &BytesStart) -> Result { + let mut cblock: Option = None; + let mut data: Option> = None; + + for a in e.attributes() { + let kv = a.unwrap(); + match kv.key { + b"cache_block" => cblock = Some(u32_val(&kv)?), + b"data" => data = Some(decode(bytes_val(&kv))?), + _ => return bad_attr("mapping", kv.key), + } + } + + let tag = "hint"; + + Ok(Hint { + cblock: check_attr(tag, "cache_block", cblock)?, + data: check_attr(tag, "data", data)?, + }) +} + +fn handle_event(reader: &mut Reader, buf: &mut Vec, visitor: &mut M) -> Result +where + R: Read + BufRead, + M: MetadataVisitor, +{ + match reader.read_event(buf) { + Ok(Event::Start(ref e)) => match e.name() { + b"superblock" => visitor.superblock_b(&parse_superblock(e)?), + b"mappings" => visitor.mappings_b(), + b"hints" => visitor.hints_b(), + _ => todo!(), + }, + Ok(Event::End(ref e)) => match e.name() { + b"superblock" => visitor.superblock_e(), + b"mappings" => visitor.mappings_e(), + b"hints" => visitor.hints_e(), + _ => todo!(), + }, + Ok(Event::Empty(ref e)) => match e.name() { + b"mapping" => visitor.mapping(&parse_mapping(e)?), + b"hint" => visitor.hint(&parse_hint(e)?), + _ => todo!(), + }, + Ok(Event::Text(_)) => Ok(Visit::Continue), + Ok(Event::Comment(_)) => Ok(Visit::Continue), + Ok(Event::Eof) => { + visitor.eof()?; + Ok(Visit::Stop) + } + Ok(_) => todo!(), + Err(e) => Err(anyhow!("{:?}", e)), + } +} + +pub fn read(input: R, visitor: &mut M) -> Result<()> +where + R: Read, + M: MetadataVisitor, +{ + let input = BufReader::new(input); + let mut reader = Reader::from_reader(input); + + reader.trim_text(true); + let mut buf = Vec::new(); + + while let Visit::Continue = handle_event(&mut reader, &mut buf, visitor)? {} + Ok(()) +} diff --git a/src/checksum.rs b/src/checksum.rs index 01d8063..f2aaa72 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -5,8 +5,9 @@ use crc32c::crc32c; use std::io::Cursor; const BLOCK_SIZE: u64 = 4096; -#[allow(dead_code)] -const SUPERBLOCK_CSUM_XOR: u32 = 160774; +const THIN_SUPERBLOCK_CSUM_XOR: u32 = 160774; +const CACHE_SUPERBLOCK_CSUM_XOR: u32 = 9031977; +const ERA_SUPERBLOCK_CSUM_XOR: u32 = 146538381; const BITMAP_CSUM_XOR: u32 = 240779; const INDEX_CSUM_XOR: u32 = 160478; const BTREE_CSUM_XOR: u32 = 121107; @@ -18,8 +19,11 @@ fn checksum(buf: &[u8]) -> u32 { #[derive(Debug, PartialEq)] #[allow(clippy::upper_case_acronyms)] +#[allow(non_camel_case_types)] pub enum BT { - SUPERBLOCK, + THIN_SUPERBLOCK, + CACHE_SUPERBLOCK, + ERA_SUPERBLOCK, NODE, INDEX, BITMAP, @@ -39,7 +43,9 @@ pub fn metadata_block_type(buf: &[u8]) -> BT { let btype = csum ^ sum_on_disk; match btype { - SUPERBLOCK_CSUM_XOR => BT::SUPERBLOCK, + THIN_SUPERBLOCK_CSUM_XOR => BT::THIN_SUPERBLOCK, + CACHE_SUPERBLOCK_CSUM_XOR => BT::CACHE_SUPERBLOCK, + ERA_SUPERBLOCK_CSUM_XOR => BT::ERA_SUPERBLOCK, BTREE_CSUM_XOR => BT::NODE, BITMAP_CSUM_XOR => BT::BITMAP, INDEX_CSUM_XOR => BT::INDEX, @@ -55,7 +61,9 @@ pub fn write_checksum(buf: &mut [u8], kind: BT) -> Result<()> { use BT::*; let salt = match kind { - SUPERBLOCK => SUPERBLOCK_CSUM_XOR, + THIN_SUPERBLOCK => THIN_SUPERBLOCK_CSUM_XOR, + CACHE_SUPERBLOCK => CACHE_SUPERBLOCK_CSUM_XOR, + ERA_SUPERBLOCK => ERA_SUPERBLOCK_CSUM_XOR, NODE => BTREE_CSUM_XOR, BITMAP => BITMAP_CSUM_XOR, INDEX => INDEX_CSUM_XOR, diff --git a/src/io_engine.rs b/src/io_engine.rs index 89b38d6..01157ec 100644 --- a/src/io_engine.rs +++ b/src/io_engine.rs @@ -17,6 +17,7 @@ use crate::file_utils; //------------------------------------------ pub const BLOCK_SIZE: usize = 4096; +pub const SECTOR_SHIFT: usize = 9; const ALIGN: usize = 4096; #[derive(Clone, Debug)] diff --git a/src/lib.rs b/src/lib.rs index 0ff3841..8398379 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,3 +27,4 @@ pub mod shrink; pub mod thin; pub mod version; pub mod write_batcher; +pub mod xml; diff --git a/src/math.rs b/src/math.rs index a3901b3..4fc4fed 100644 --- a/src/math.rs +++ b/src/math.rs @@ -1,7 +1,47 @@ -pub fn div_up(v: usize, divisor: usize) -> usize { - v / divisor + (v % divisor != 0) as usize +use std::cmp::PartialEq; +use std::ops::{Add, Div, Rem}; + +//----------------------------------------- + +pub trait Integer: + Sized + Copy + Add + Div + Rem + PartialEq +{ + fn zero() -> Self; + fn one() -> Self; } -pub fn div_down(v: usize, divisor: usize) -> usize { +pub fn div_up(v: T, divisor: T) -> T { + if v % divisor != Integer::zero() { + v / divisor + Integer::one() + } else { + v / divisor + } +} + +pub fn div_down(v: T, divisor: T) -> T { v / divisor } + +//----------------------------------------- + +impl Integer for usize { + fn zero() -> Self { + 0 + } + + fn one() -> Self { + 1 + } +} + +impl Integer for u64 { + fn zero() -> Self { + 0 + } + + fn one() -> Self { + 1 + } +} + +//----------------------------------------- diff --git a/src/pack/toplevel.rs b/src/pack/toplevel.rs index 98b4dcf..d4adf7d 100644 --- a/src/pack/toplevel.rs +++ b/src/pack/toplevel.rs @@ -205,7 +205,9 @@ where fn pack_block(w: &mut W, kind: BT, buf: &[u8]) -> Result<()> { match kind { - BT::SUPERBLOCK => pack_superblock(w, buf).context("unable to pack superblock")?, + BT::THIN_SUPERBLOCK | BT::CACHE_SUPERBLOCK | BT::ERA_SUPERBLOCK => { + pack_superblock(w, buf).context("unable to pack superblock")? + } BT::NODE => pack_btree_node(w, buf).context("unable to pack btree node")?, BT::INDEX => pack_index(w, buf).context("unable to pack space map index")?, BT::BITMAP => pack_bitmap(w, buf).context("unable to pack space map bitmap")?, diff --git a/src/pdata/array.rs b/src/pdata/array.rs index f459d1f..0485d2a 100644 --- a/src/pdata/array.rs +++ b/src/pdata/array.rs @@ -1,3 +1,4 @@ +use byteorder::{LittleEndian, WriteBytesExt}; use nom::{multi::count, number::complete::*, IResult}; use std::fmt; use thiserror::Error; @@ -5,7 +6,7 @@ use thiserror::Error; use crate::checksum; use crate::io_engine::BLOCK_SIZE; use crate::pdata::btree; -use crate::pdata::unpack::Unpack; +use crate::pdata::unpack::{Pack, Unpack}; //------------------------------------------ @@ -44,11 +45,31 @@ impl Unpack for ArrayBlockHeader { } } +impl Pack for ArrayBlockHeader { + fn pack(&self, w: &mut W) -> anyhow::Result<()> { + // csum needs to be calculated right for the whole metadata block. + w.write_u32::(0)?; + w.write_u32::(self.max_entries)?; + w.write_u32::(self.nr_entries)?; + w.write_u32::(self.value_size)?; + w.write_u64::(self.blocknr)?; + Ok(()) + } +} + +//------------------------------------------ + pub struct ArrayBlock { pub header: ArrayBlockHeader, pub values: Vec, } +impl ArrayBlock { + pub fn set_block(&mut self, b: u64) { + self.header.blocknr = b; + } +} + //------------------------------------------ #[derive(Error, Clone, Debug)] diff --git a/src/pdata/array_builder.rs b/src/pdata/array_builder.rs new file mode 100644 index 0000000..8b7d540 --- /dev/null +++ b/src/pdata/array_builder.rs @@ -0,0 +1,212 @@ +use anyhow::Result; +use byteorder::WriteBytesExt; +use std::collections::VecDeque; +use std::io::Cursor; + +use crate::checksum; +use crate::io_engine::*; +use crate::pdata::array::*; +use crate::pdata::btree_builder::*; +use crate::pdata::unpack::*; +use crate::write_batcher::*; + +//------------------------------------------ + +pub struct ArrayBlockBuilder { + array_io: ArrayIO, + max_entries_per_block: usize, + values: VecDeque<(u64, V)>, + array_blocks: Vec, + nr_entries: u64, + nr_emitted: u64, + nr_queued: u64, +} + +pub struct ArrayBuilder { + block_builder: ArrayBlockBuilder, +} + +struct ArrayIO { + dummy: std::marker::PhantomData, +} + +struct WriteResult { + loc: u64, +} + +//------------------------------------------ + +fn calc_max_entries() -> usize { + (BLOCK_SIZE - ArrayBlockHeader::disk_size() as usize) / V::disk_size() as usize +} + +//------------------------------------------ + +impl ArrayBlockBuilder { + pub fn new(nr_entries: u64) -> ArrayBlockBuilder { + ArrayBlockBuilder { + array_io: ArrayIO::new(), + max_entries_per_block: calc_max_entries::(), + values: VecDeque::new(), + array_blocks: Vec::new(), + nr_entries, + nr_emitted: 0, + nr_queued: 0, + } + } + + pub fn push_value(&mut self, w: &mut WriteBatcher, index: u64, v: V) -> Result<()> { + assert!(index >= self.nr_emitted + self.nr_queued); + assert!(index < self.nr_entries); + + self.values.push_back((index, v)); + self.nr_queued = index - self.nr_emitted + 1; + + if self.nr_queued > self.max_entries_per_block as u64 { + self.emit_blocks(w)?; + } + + Ok(()) + } + + pub fn complete(mut self, w: &mut WriteBatcher) -> Result> { + if self.nr_emitted + self.nr_queued < self.nr_entries { + // FIXME: flushing with a default values looks confusing + self.push_value(w, self.nr_entries - 1, Default::default())?; + } + self.emit_all(w)?; + Ok(self.array_blocks) + } + + /// Emit all the remaining queued values + fn emit_all(&mut self, w: &mut WriteBatcher) -> Result<()> { + match self.nr_queued { + 0 => { + // There's nothing to emit + Ok(()) + } + n if n <= self.max_entries_per_block as u64 => self.emit_values(w), + _ => { + panic!( + "There shouldn't be more than {} queued values", + self.max_entries_per_block + ); + } + } + } + + /// Emit one or more fully utilized array blocks + fn emit_blocks(&mut self, w: &mut WriteBatcher) -> Result<()> { + while self.nr_queued > self.max_entries_per_block as u64 { + self.emit_values(w)?; + } + Ok(()) + } + + /// Emit an array block with the queued values + fn emit_values(&mut self, w: &mut WriteBatcher) -> Result<()> { + let mut values = Vec::::with_capacity(self.max_entries_per_block); + let mut nr_free = self.max_entries_per_block; + + while !self.values.is_empty() && nr_free > 0 { + let len = self.values.front().unwrap().0 - self.nr_emitted + 1; + if len <= nr_free as u64 { + let (_, v) = self.values.pop_front().unwrap(); + if len > 1 { + values.resize_with(values.len() + len as usize - 1, Default::default); + } + values.push(v); + nr_free -= len as usize; + self.nr_emitted += len; + self.nr_queued -= len; + } else { + values.resize_with(values.len() + nr_free as usize, Default::default); + self.nr_emitted += nr_free as u64; + self.nr_queued -= nr_free as u64; + nr_free = 0; + } + } + + let wresult = self.array_io.write(w, values)?; + + self.array_blocks.push(wresult.loc); + + Ok(()) + } +} + +//------------------------------------------ + +impl ArrayBuilder { + pub fn new(nr_entries: u64) -> ArrayBuilder { + ArrayBuilder { + block_builder: ArrayBlockBuilder::::new(nr_entries), + } + } + + pub fn push_value(&mut self, w: &mut WriteBatcher, index: u64, v: V) -> Result<()> { + self.block_builder.push_value(w, index, v) + } + + pub fn complete(self, w: &mut WriteBatcher) -> Result { + let blocks = self.block_builder.complete(w)?; + let mut index_builder = Builder::::new(Box::new(NoopRC {})); + + for (i, b) in blocks.iter().enumerate() { + index_builder.push_value(w, i as u64, *b)?; + } + index_builder.complete(w) + } +} + +//------------------------------------------ + +impl ArrayIO { + pub fn new() -> ArrayIO { + ArrayIO { + dummy: std::marker::PhantomData, + } + } + + fn write(&self, w: &mut WriteBatcher, values: Vec) -> Result { + let header = ArrayBlockHeader { + csum: 0, + max_entries: calc_max_entries::() as u32, + nr_entries: values.len() as u32, + value_size: V::disk_size(), + blocknr: 0, + }; + + let ablock = ArrayBlock { header, values }; + + write_array_block(w, ablock) + } +} + +fn write_array_block( + w: &mut WriteBatcher, + mut ablock: ArrayBlock, +) -> Result { + let b = w.alloc()?; + ablock.set_block(b.loc); + + let mut cursor = Cursor::new(b.get_data()); + pack_array_block(&ablock, &mut cursor)?; + let loc = b.loc; + w.write(b, checksum::BT::ARRAY)?; + + Ok(WriteResult { loc }) +} + +fn pack_array_block( + ablock: &ArrayBlock, + w: &mut W, +) -> Result<()> { + ablock.header.pack(w)?; + for v in ablock.values.iter() { + v.pack(w)?; + } + Ok(()) +} + +//------------------------------------------ diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs index 56e8f2c..21ce07c 100644 --- a/src/pdata/mod.rs +++ b/src/pdata/mod.rs @@ -1,4 +1,5 @@ pub mod array; +pub mod array_builder; pub mod array_walker; pub mod bitset; pub mod btree; @@ -8,5 +9,7 @@ pub mod btree_merge; pub mod btree_walker; pub mod space_map; pub mod space_map_checker; +pub mod space_map_common; pub mod space_map_disk; +pub mod space_map_metadata; pub mod unpack; diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 5f1f2f0..7c138bf 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -103,6 +103,7 @@ where if self.counts[b as usize] == V::from(0u8) { self.counts[b as usize] = V::from(1u8); self.first_free = b + 1; + self.nr_allocated += 1; return Ok(Some(b)); } } @@ -132,6 +133,16 @@ pub fn core_sm_without_mutex(nr_entries: u64, max_count: u32) -> Box Result> { + let nr_blocks = src.get_nr_blocks()?; + let mut dest = Box::new(CoreSpaceMap::::new(nr_blocks)); + for i in 0..nr_blocks { + dest.set(i, src.get(i)?)?; + } + Ok(dest) +} + //------------------------------------------ // This in core space map can only count to one, useful when walking diff --git a/src/pdata/space_map_checker.rs b/src/pdata/space_map_checker.rs index 9a944a1..d0a9f47 100644 --- a/src/pdata/space_map_checker.rs +++ b/src/pdata/space_map_checker.rs @@ -7,7 +7,8 @@ use crate::io_engine::IoEngine; use crate::pdata::btree::{self, *}; use crate::pdata::btree_walker::*; use crate::pdata::space_map::*; -use crate::pdata::space_map_disk::*; +use crate::pdata::space_map_common::*; +use crate::pdata::space_map_metadata::*; use crate::pdata::unpack::*; use crate::report::Report; diff --git a/src/pdata/space_map_common.rs b/src/pdata/space_map_common.rs new file mode 100644 index 0000000..e2f8918 --- /dev/null +++ b/src/pdata/space_map_common.rs @@ -0,0 +1,257 @@ +use anyhow::{anyhow, Result}; +use byteorder::{LittleEndian, WriteBytesExt}; +use nom::{number::complete::*, IResult}; +use std::io::Cursor; + +use crate::checksum; +use crate::io_engine::*; +use crate::math::*; +use crate::pdata::btree_builder::*; +use crate::pdata::space_map::*; +use crate::pdata::unpack::*; +use crate::write_batcher::*; + +//------------------------------------------ + +pub const ENTRIES_PER_BITMAP: usize = WORDS_PER_BITMAP * 8 * ENTRIES_PER_BYTE; +const WORDS_PER_BITMAP: usize = (BLOCK_SIZE - 16) / 8; +const ENTRIES_PER_BYTE: usize = 4; + +//------------------------------------------ + +#[derive(Clone, Copy, Debug)] +pub struct IndexEntry { + pub blocknr: u64, + pub nr_free: u32, + pub none_free_before: u32, +} + +impl Unpack for IndexEntry { + fn disk_size() -> u32 { + 16 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], IndexEntry> { + let (i, blocknr) = le_u64(i)?; + let (i, nr_free) = le_u32(i)?; + let (i, none_free_before) = le_u32(i)?; + + Ok(( + i, + IndexEntry { + blocknr, + nr_free, + none_free_before, + }, + )) + } +} + +impl Pack for IndexEntry { + fn pack(&self, w: &mut W) -> Result<()> { + w.write_u64::(self.blocknr)?; + w.write_u32::(self.nr_free)?; + w.write_u32::(self.none_free_before)?; + Ok(()) + } +} + +//------------------------------------------ + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BitmapEntry { + Small(u8), + Overflow, +} + +#[derive(Debug)] +pub struct Bitmap { + pub blocknr: u64, + pub entries: Vec, +} + +impl Unpack for Bitmap { + fn disk_size() -> u32 { + BLOCK_SIZE as u32 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (i, _csum) = le_u32(data)?; + let (i, _not_used) = le_u32(i)?; + let (mut i, blocknr) = le_u64(i)?; + + let header_size = 16; + let nr_words = (BLOCK_SIZE - header_size) / 8; + let mut entries = Vec::with_capacity(nr_words * 32); + for _w in 0..nr_words { + let (tmp, mut word) = le_u64(i)?; + + for _b in 0..32 { + let val = word & 0x3; + word >>= 2; + + // The bits are stored with the high bit at b * 2 + 1, + // and low at b *2. So we have to interpret this val. + entries.push(match val { + 0 => BitmapEntry::Small(0), + 1 => BitmapEntry::Small(2), + 2 => BitmapEntry::Small(1), + _ => BitmapEntry::Overflow, + }); + } + + i = tmp; + } + + Ok((i, Bitmap { blocknr, entries })) + } +} + +impl Pack for Bitmap { + fn pack(&self, out: &mut W) -> Result<()> { + use BitmapEntry::*; + + out.write_u32::(0)?; + out.write_u32::(0)?; + out.write_u64::(self.blocknr)?; + + for chunk in self.entries.chunks(32) { + let mut w = 0u64; + for e in chunk { + w >>= 2; + match e { + Small(0) => {} + Small(1) => { + w |= 0x2 << 62; + } + Small(2) => { + w |= 0x1 << 62; + } + Small(_) => { + return Err(anyhow!("Bad small value in bitmap entry")); + } + Overflow => { + w |= 0x3 << 62; + } + } + } + + u64::pack(&w, out)?; + } + + Ok(()) + } +} + +//------------------------------------------ + +#[derive(Debug)] +pub struct SMRoot { + pub nr_blocks: u64, + pub nr_allocated: u64, + pub bitmap_root: u64, + pub ref_count_root: u64, +} + +impl Unpack for SMRoot { + fn disk_size() -> u32 { + 32 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], Self> { + let (i, nr_blocks) = le_u64(i)?; + let (i, nr_allocated) = le_u64(i)?; + let (i, bitmap_root) = le_u64(i)?; + let (i, ref_count_root) = le_u64(i)?; + + Ok(( + i, + SMRoot { + nr_blocks, + nr_allocated, + bitmap_root, + ref_count_root, + }, + )) + } +} + +pub fn unpack_root(data: &[u8]) -> Result { + match SMRoot::unpack(data) { + Err(_e) => Err(anyhow!("couldn't parse SMRoot")), + Ok((_i, v)) => Ok(v), + } +} + +impl Pack for SMRoot { + fn pack(&self, w: &mut W) -> Result<()> { + w.write_u64::(self.nr_blocks)?; + w.write_u64::(self.nr_allocated)?; + w.write_u64::(self.bitmap_root)?; + w.write_u64::(self.ref_count_root)?; + + Ok(()) + } +} + +//------------------------------------------ + +pub fn write_common(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result<(Vec, u64)> { + use BitmapEntry::*; + + let mut index_entries = Vec::new(); + let mut overflow_builder: Builder = Builder::new(Box::new(NoopRC {})); + + // how many bitmaps do we need? + for bm in 0..div_up(sm.get_nr_blocks()? as usize, ENTRIES_PER_BITMAP) { + let mut entries = Vec::with_capacity(ENTRIES_PER_BITMAP); + let mut first_free: Option = None; + let mut nr_free: u32 = 0; + for i in 0..ENTRIES_PER_BITMAP { + let b: u64 = ((bm * ENTRIES_PER_BITMAP) as u64) + i as u64; + if b >= sm.get_nr_blocks()? { + break; + } + let rc = sm.get(b)?; + let e = match rc { + 0 => { + nr_free += 1; + if first_free.is_none() { + first_free = Some(i as u32); + } + Small(0) + } + 1 => Small(1), + 2 => Small(2), + _ => { + overflow_builder.push_value(w, b as u64, rc)?; + Overflow + } + }; + entries.push(e); + } + + // allocate a new block + let b = w.alloc()?; + let mut cursor = Cursor::new(b.get_data()); + + // write the bitmap to it + let blocknr = b.loc; + let bitmap = Bitmap { blocknr, entries }; + bitmap.pack(&mut cursor)?; + w.write(b, checksum::BT::BITMAP)?; + + // Insert into the index tree + let ie = IndexEntry { + blocknr, + nr_free, + none_free_before: first_free.unwrap_or(ENTRIES_PER_BITMAP as u32), + }; + index_entries.push(ie); + } + + let ref_count_root = overflow_builder.complete(w)?; + Ok((index_entries, ref_count_root)) +} + +//------------------------------------------ diff --git a/src/pdata/space_map_disk.rs b/src/pdata/space_map_disk.rs index 2dee628..88e1749 100644 --- a/src/pdata/space_map_disk.rs +++ b/src/pdata/space_map_disk.rs @@ -1,301 +1,11 @@ -use anyhow::{anyhow, Result}; -use byteorder::{LittleEndian, WriteBytesExt}; -use nom::{number::complete::*, IResult}; -use std::collections::BTreeMap; -use std::io::Cursor; +use anyhow::Result; -use crate::checksum; -use crate::io_engine::*; -use crate::math::*; use crate::pdata::btree_builder::*; use crate::pdata::space_map::*; -use crate::pdata::unpack::*; +use crate::pdata::space_map_common::*; use crate::write_batcher::*; -//-------------------------------- - -const MAX_METADATA_BITMAPS: usize = 255; -// const MAX_METADATA_BLOCKS: u64 = 255 * ((1 << 14) - 64); -const ENTRIES_PER_BYTE: usize = 4; -const ENTRIES_PER_BITMAP: usize = WORDS_PER_BITMAP * 8 * ENTRIES_PER_BYTE; - -//-------------------------------- - -#[derive(Clone, Copy, Debug)] -pub struct IndexEntry { - pub blocknr: u64, - pub nr_free: u32, - pub none_free_before: u32, -} - -impl Unpack for IndexEntry { - fn disk_size() -> u32 { - 16 - } - - fn unpack(i: &[u8]) -> IResult<&[u8], IndexEntry> { - let (i, blocknr) = le_u64(i)?; - let (i, nr_free) = le_u32(i)?; - let (i, none_free_before) = le_u32(i)?; - - Ok(( - i, - IndexEntry { - blocknr, - nr_free, - none_free_before, - }, - )) - } -} - -impl Pack for IndexEntry { - fn pack(&self, w: &mut W) -> Result<()> { - w.write_u64::(self.blocknr)?; - w.write_u32::(self.nr_free)?; - w.write_u32::(self.none_free_before)?; - Ok(()) - } -} - -//-------------------------------- - -pub struct MetadataIndex { - pub blocknr: u64, - pub indexes: Vec, -} - -impl Unpack for MetadataIndex { - fn disk_size() -> u32 { - BLOCK_SIZE as u32 - } - - fn unpack(i: &[u8]) -> IResult<&[u8], MetadataIndex> { - // FIXME: check the checksum - let (i, _csum) = le_u32(i)?; - let (i, _padding) = le_u32(i)?; - let (i, blocknr) = le_u64(i)?; - let (i, indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?; - - Ok((i, MetadataIndex { blocknr, indexes })) - } -} - -impl Pack for MetadataIndex { - fn pack(&self, w: &mut W) -> Result<()> { - w.write_u32::(0)?; // csum - w.write_u32::(0)?; // padding - w.write_u64::(self.blocknr)?; - - assert!(self.indexes.len() <= MAX_METADATA_BITMAPS); - - for ie in &self.indexes { - ie.pack(w)?; - } - - Ok(()) - } -} - -//-------------------------------- - -const WORDS_PER_BITMAP: usize = (BLOCK_SIZE - 16) / 8; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BitmapEntry { - Small(u8), - Overflow, -} - -#[derive(Debug)] -pub struct Bitmap { - pub blocknr: u64, - pub entries: Vec, -} - -impl Unpack for Bitmap { - fn disk_size() -> u32 { - BLOCK_SIZE as u32 - } - - fn unpack(data: &[u8]) -> IResult<&[u8], Self> { - let (i, _csum) = le_u32(data)?; - let (i, _not_used) = le_u32(i)?; - let (mut i, blocknr) = le_u64(i)?; - - let header_size = 16; - let nr_words = (BLOCK_SIZE - header_size) / 8; - let mut entries = Vec::with_capacity(nr_words * 32); - for _w in 0..nr_words { - let (tmp, mut word) = le_u64(i)?; - - for _b in 0..32 { - let val = word & 0x3; - word >>= 2; - - // The bits are stored with the high bit at b * 2 + 1, - // and low at b *2. So we have to interpret this val. - entries.push(match val { - 0 => BitmapEntry::Small(0), - 1 => BitmapEntry::Small(2), - 2 => BitmapEntry::Small(1), - _ => BitmapEntry::Overflow, - }); - } - - i = tmp; - } - - Ok((i, Bitmap { blocknr, entries })) - } -} - -impl Pack for Bitmap { - fn pack(&self, out: &mut W) -> Result<()> { - use BitmapEntry::*; - - out.write_u32::(0)?; - out.write_u32::(0)?; - out.write_u64::(self.blocknr)?; - - for chunk in self.entries.chunks(32) { - let mut w = 0u64; - for e in chunk { - w >>= 2; - match e { - Small(0) => {} - Small(1) => { - w |= 0x2 << 62; - } - Small(2) => { - w |= 0x1 << 62; - } - Small(_) => { - return Err(anyhow!("Bad small value in bitmap entry")); - } - Overflow => { - w |= 0x3 << 62; - } - } - } - - u64::pack(&w, out)?; - } - - Ok(()) - } -} - -//-------------------------------- - -#[derive(Debug)] -pub struct SMRoot { - pub nr_blocks: u64, - pub nr_allocated: u64, - pub bitmap_root: u64, - pub ref_count_root: u64, -} - -impl Unpack for SMRoot { - fn disk_size() -> u32 { - 32 - } - - fn unpack(i: &[u8]) -> IResult<&[u8], Self> { - let (i, nr_blocks) = le_u64(i)?; - let (i, nr_allocated) = le_u64(i)?; - let (i, bitmap_root) = le_u64(i)?; - let (i, ref_count_root) = le_u64(i)?; - - Ok(( - i, - SMRoot { - nr_blocks, - nr_allocated, - bitmap_root, - ref_count_root, - }, - )) - } -} - -pub fn unpack_root(data: &[u8]) -> Result { - match SMRoot::unpack(data) { - Err(_e) => Err(anyhow!("couldn't parse SMRoot")), - Ok((_i, v)) => Ok(v), - } -} - -impl Pack for SMRoot { - fn pack(&self, w: &mut W) -> Result<()> { - w.write_u64::(self.nr_blocks)?; - w.write_u64::(self.nr_allocated)?; - w.write_u64::(self.bitmap_root)?; - w.write_u64::(self.ref_count_root)?; - - Ok(()) - } -} - -//-------------------------------- - -pub fn write_common(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result<(Vec, u64)> { - use BitmapEntry::*; - - let mut index_entries = Vec::new(); - let mut overflow_builder: Builder = Builder::new(Box::new(NoopRC {})); - - // how many bitmaps do we need? - for bm in 0..div_up(sm.get_nr_blocks()? as usize, ENTRIES_PER_BITMAP) { - let mut entries = Vec::with_capacity(ENTRIES_PER_BITMAP); - let mut first_free: Option = None; - let mut nr_free: u32 = 0; - for i in 0..ENTRIES_PER_BITMAP { - let b: u64 = ((bm * ENTRIES_PER_BITMAP) as u64) + i as u64; - if b > sm.get_nr_blocks()? { - break; - } - let rc = sm.get(b)?; - let e = match rc { - 0 => { - nr_free += 1; - if first_free.is_none() { - first_free = Some(i as u32); - } - Small(0) - } - 1 => Small(1), - 2 => Small(2), - _ => { - overflow_builder.push_value(w, b as u64, rc)?; - Overflow - } - }; - entries.push(e); - } - - // allocate a new block - let b = w.alloc()?; - let mut cursor = Cursor::new(b.get_data()); - - // write the bitmap to it - let blocknr = b.loc; - let bitmap = Bitmap { blocknr, entries }; - bitmap.pack(&mut cursor)?; - w.write(b, checksum::BT::BITMAP)?; - - // Insert into the index tree - let ie = IndexEntry { - blocknr, - nr_free, - none_free_before: first_free.unwrap_or(ENTRIES_PER_BITMAP as u32), - }; - index_entries.push(ie); - } - - let ref_count_root = overflow_builder.complete(w)?; - Ok((index_entries, ref_count_root)) -} +//------------------------------------------ pub fn write_disk_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result { let (index_entries, ref_count_root) = write_common(w, sm)?; @@ -306,6 +16,7 @@ pub fn write_disk_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result } let bitmap_root = index_builder.complete(w)?; + w.flush()?; Ok(SMRoot { nr_blocks: sm.get_nr_blocks()?, @@ -315,85 +26,4 @@ pub fn write_disk_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result }) } -//---------------------------- - -fn block_to_bitmap(b: u64) -> usize { - (b / ENTRIES_PER_BITMAP as u64) as usize -} - -fn adjust_counts(w: &mut WriteBatcher, ie: &IndexEntry, allocs: &[u64]) -> Result { - use BitmapEntry::*; - - let mut first_free = ie.none_free_before; - let mut nr_free = ie.nr_free - allocs.len() as u32; - - // Read the bitmap - let bitmap_block = w.engine.read(ie.blocknr)?; - let (_, mut bitmap) = Bitmap::unpack(bitmap_block.get_data())?; - - // Update all the entries - for a in allocs { - if first_free == *a as u32 { - first_free = *a as u32 + 1; - } - - if bitmap.entries[*a as usize] == Small(0) { - nr_free -= 1; - } - - bitmap.entries[*a as usize] = Small(1); - } - - // Write the bitmap - let mut cur = Cursor::new(bitmap_block.get_data()); - bitmap.pack(&mut cur)?; - w.write(bitmap_block, checksum::BT::BITMAP)?; - - // Return the adjusted index entry - Ok(IndexEntry { - blocknr: ie.blocknr, - nr_free, - none_free_before: first_free, - }) -} - -pub fn write_metadata_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result { - w.clear_allocations(); - let (mut indexes, ref_count_root) = write_common(w, sm)?; - - let bitmap_root = w.alloc()?; - - // Now we need to patch up the counts for the metadata that was used for storing - // the space map itself. These ref counts all went from 0 to 1. - let allocations = w.clear_allocations(); - - // Sort the allocations by bitmap - let mut by_bitmap = BTreeMap::new(); - for b in allocations { - let bitmap = block_to_bitmap(b); - (*by_bitmap.entry(bitmap).or_insert_with(Vec::new)).push(b % ENTRIES_PER_BITMAP as u64); - } - - for (bitmap, allocs) in by_bitmap { - indexes[bitmap] = adjust_counts(w, &indexes[bitmap], &allocs)?; - } - - // Write out the metadata index - let metadata_index = MetadataIndex { - blocknr: bitmap_root.loc, - indexes, - }; - let mut cur = Cursor::new(bitmap_root.get_data()); - metadata_index.pack(&mut cur)?; - let loc = bitmap_root.loc; - w.write(bitmap_root, checksum::BT::INDEX)?; - - Ok(SMRoot { - nr_blocks: sm.get_nr_blocks()?, - nr_allocated: sm.get_nr_allocated()?, - bitmap_root: loc, - ref_count_root, - }) -} - -//-------------------------------- +//------------------------------------------ diff --git a/src/pdata/space_map_metadata.rs b/src/pdata/space_map_metadata.rs new file mode 100644 index 0000000..4567b05 --- /dev/null +++ b/src/pdata/space_map_metadata.rs @@ -0,0 +1,139 @@ +use anyhow::Result; +use byteorder::{LittleEndian, WriteBytesExt}; +use nom::{number::complete::*, IResult}; +use std::collections::BTreeMap; +use std::io::Cursor; + +use crate::checksum; +use crate::io_engine::*; +use crate::pdata::space_map::*; +use crate::pdata::space_map_common::*; +use crate::pdata::unpack::*; +use crate::write_batcher::*; + +//------------------------------------------ + +const MAX_METADATA_BITMAPS: usize = 255; + +//------------------------------------------ + +pub struct MetadataIndex { + pub blocknr: u64, + pub indexes: Vec, +} + +impl Unpack for MetadataIndex { + fn disk_size() -> u32 { + BLOCK_SIZE as u32 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], MetadataIndex> { + // FIXME: check the checksum + let (i, _csum) = le_u32(i)?; + let (i, _padding) = le_u32(i)?; + let (i, blocknr) = le_u64(i)?; + let (i, indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?; + + Ok((i, MetadataIndex { blocknr, indexes })) + } +} + +impl Pack for MetadataIndex { + fn pack(&self, w: &mut W) -> Result<()> { + w.write_u32::(0)?; // csum + w.write_u32::(0)?; // padding + w.write_u64::(self.blocknr)?; + + assert!(self.indexes.len() <= MAX_METADATA_BITMAPS); + + for ie in &self.indexes { + ie.pack(w)?; + } + + Ok(()) + } +} + +//------------------------------------------ + +fn block_to_bitmap(b: u64) -> usize { + (b / ENTRIES_PER_BITMAP as u64) as usize +} + +fn adjust_counts(w: &mut WriteBatcher, ie: &IndexEntry, allocs: &[u64]) -> Result { + use BitmapEntry::*; + + let mut first_free = ie.none_free_before; + let mut nr_free = ie.nr_free - allocs.len() as u32; + + // Read the bitmap + let bitmap_block = w.engine.read(ie.blocknr)?; + let (_, mut bitmap) = Bitmap::unpack(bitmap_block.get_data())?; + + // Update all the entries + for a in allocs { + if first_free == *a as u32 { + first_free = *a as u32 + 1; + } + + if bitmap.entries[*a as usize] == Small(0) { + nr_free -= 1; + } + + bitmap.entries[*a as usize] = Small(1); + } + + // Write the bitmap + let mut cur = Cursor::new(bitmap_block.get_data()); + bitmap.pack(&mut cur)?; + w.write(bitmap_block, checksum::BT::BITMAP)?; + + // Return the adjusted index entry + Ok(IndexEntry { + blocknr: ie.blocknr, + nr_free, + none_free_before: first_free, + }) +} + +pub fn write_metadata_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result { + w.clear_allocations(); + let (mut indexes, ref_count_root) = write_common(w, sm)?; + + let bitmap_root = w.alloc()?; + + // Now we need to patch up the counts for the metadata that was used for storing + // the space map itself. These ref counts all went from 0 to 1. + let allocations = w.clear_allocations(); + + // Sort the allocations by bitmap + let mut by_bitmap = BTreeMap::new(); + for b in allocations { + let bitmap = block_to_bitmap(b); + (*by_bitmap.entry(bitmap).or_insert_with(Vec::new)).push(b % ENTRIES_PER_BITMAP as u64); + } + + for (bitmap, allocs) in by_bitmap { + indexes[bitmap] = adjust_counts(w, &indexes[bitmap], &allocs)?; + } + + // Write out the metadata index + let metadata_index = MetadataIndex { + blocknr: bitmap_root.loc, + indexes, + }; + let mut cur = Cursor::new(bitmap_root.get_data()); + metadata_index.pack(&mut cur)?; + let loc = bitmap_root.loc; + w.write(bitmap_root, checksum::BT::INDEX)?; + w.flush()?; + + Ok(SMRoot { + nr_blocks: sm.get_nr_blocks()?, + nr_allocated: sm.get_nr_allocated()?, + bitmap_root: loc, + ref_count_root, + }) +} + +//------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index d3aba19..1c5836e 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -10,7 +10,7 @@ use crate::pdata::btree::{self, *}; use crate::pdata::btree_walker::*; use crate::pdata::space_map::*; use crate::pdata::space_map_checker::*; -use crate::pdata::space_map_disk::*; +use crate::pdata::space_map_common::*; use crate::pdata::unpack::*; use crate::report::*; use crate::thin::block_time::*; diff --git a/src/thin/dump.rs b/src/thin/dump.rs index f1c9080..d8c796f 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -11,7 +11,7 @@ use crate::pdata::btree::{self, *}; use crate::pdata::btree_leaf_walker::*; use crate::pdata::btree_walker::*; use crate::pdata::space_map::*; -use crate::pdata::space_map_disk::*; +use crate::pdata::space_map_common::*; use crate::pdata::unpack::*; use crate::report::*; use crate::thin::block_time::*; diff --git a/src/thin/restore.rs b/src/thin/restore.rs index 612dd93..7f1bfa7 100644 --- a/src/thin/restore.rs +++ b/src/thin/restore.rs @@ -2,12 +2,17 @@ use anyhow::{anyhow, Result}; use std::collections::BTreeMap; use std::fs::OpenOptions; +use std::io::Cursor; +use std::ops::Deref; use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use crate::io_engine::*; use crate::pdata::btree_builder::*; use crate::pdata::space_map::*; +use crate::pdata::space_map_disk::*; +use crate::pdata::space_map_metadata::*; +use crate::pdata::unpack::Pack; use crate::report::*; use crate::thin::block_time::*; use crate::thin::device_detail::*; @@ -17,6 +22,25 @@ use crate::write_batcher::*; //------------------------------------------ +struct MappingRC { + sm: Arc>, +} + +impl RefCounter for MappingRC { + fn get(&self, v: &BlockTime) -> Result { + return self.sm.lock().unwrap().get(v.block); + } + fn inc(&mut self, v: &BlockTime) -> Result<()> { + self.sm.lock().unwrap().inc(v.block, 1) + } + fn dec(&mut self, v: &BlockTime) -> Result<()> { + self.sm.lock().unwrap().dec(v.block)?; + Ok(()) + } +} + +//------------------------------------------ + enum MappedSection { Def(String), Dev(u32), @@ -31,9 +55,12 @@ impl std::fmt::Display for MappedSection { } } +//------------------------------------------ + struct Pass1Result { - sb: Option, + sb: xml::Superblock, devices: BTreeMap)>, + data_sm: Arc>, } struct Pass1<'a> { @@ -45,7 +72,9 @@ struct Pass1<'a> { // The builder for the current shared sub tree or device map: Option<(MappedSection, NodeBuilder)>, - result: Pass1Result, + sb: Option, + devices: BTreeMap)>, + data_sm: Option>>, } impl<'a> Pass1<'a> { @@ -55,15 +84,21 @@ impl<'a> Pass1<'a> { current_dev: None, sub_trees: BTreeMap::new(), map: None, - result: Pass1Result { - sb: None, - devices: BTreeMap::new(), - }, + sb: None, + devices: BTreeMap::new(), + data_sm: None, } } - fn get_result(self) -> Pass1Result { - self.result + fn get_result(self) -> Result { + if self.sb.is_none() { + return Err(anyhow!("No superblock found in xml file")); + } + Ok(Pass1Result { + sb: self.sb.unwrap(), + devices: self.devices, + data_sm: self.data_sm.unwrap(), + }) } fn begin_section(&mut self, section: MappedSection) -> Result { @@ -75,7 +110,9 @@ impl<'a> Pass1<'a> { return Err(anyhow!(msg)); } - let value_rc = Box::new(NoopRC {}); + let value_rc = Box::new(MappingRC { + sm: self.data_sm.as_ref().unwrap().clone(), + }); let leaf_builder = NodeBuilder::new(Box::new(LeafIO {}), value_rc); self.map = Some((section, leaf_builder)); @@ -97,7 +134,8 @@ impl<'a> Pass1<'a> { impl<'a> MetadataVisitor for Pass1<'a> { fn superblock_b(&mut self, sb: &xml::Superblock) -> Result { - self.result.sb = Some(sb.clone()); + self.sb = Some(sb.clone()); + self.data_sm = Some(core_sm(sb.nr_data_blocks, u32::MAX)); self.w.alloc()?; Ok(Visit::Continue) } @@ -132,7 +170,7 @@ impl<'a> MetadataVisitor for Pass1<'a> { fn device_e(&mut self) -> Result { if let Some(detail) = self.current_dev.take() { if let (MappedSection::Dev(thin_id), nodes) = self.end_section()? { - self.result.devices.insert(thin_id, (detail, nodes)); + self.devices.insert(thin_id, (detail, nodes)); Ok(Visit::Continue) } else { Err(anyhow!("internal error, couldn't find device details")) @@ -191,13 +229,29 @@ impl<'a> MetadataVisitor for Pass1<'a> { } //------------------------------------------ -/* + /// Writes a data space map to disk. Returns the space map root that needs /// to be written to the superblock. -fn build_data_sm(batcher: WriteBatcher, sm: Box) -> Result> { +fn build_data_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result> { + let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; + let mut cur = Cursor::new(&mut sm_root); + let r = write_disk_sm(w, sm)?; + r.pack(&mut cur)?; + Ok(sm_root) +} + +/// Writes the metadata space map to disk. Returns the space map root that needs +/// to be written to the superblock. +fn build_metadata_sm(w: &mut WriteBatcher) -> Result> { + let mut sm_root = vec![0u8; SPACE_MAP_ROOT_SIZE]; + let mut cur = Cursor::new(&mut sm_root); + let sm_without_meta = clone_space_map(w.sm.lock().unwrap().deref())?; + let r = write_metadata_sm(w, sm_without_meta.deref())?; + r.pack(&mut cur)?; + + Ok(sm_root) } -*/ //------------------------------------------ @@ -246,7 +300,7 @@ pub fn restore(opts: ThinRestoreOptions) -> Result<()> { let mut w = WriteBatcher::new(ctx.engine.clone(), sm.clone(), ctx.engine.get_batch_size()); let mut pass = Pass1::new(&mut w); xml::read(input, &mut pass)?; - let pass = pass.get_result(); + let pass = pass.get_result()?; // Build the device details tree. let mut details_builder: Builder = Builder::new(Box::new(NoopRC {})); @@ -274,33 +328,28 @@ pub fn restore(opts: ThinRestoreOptions) -> Result<()> { let mapping_root = builder.complete(&mut w)?; // Build data space map + let data_sm_root = build_data_sm(&mut w, pass.data_sm.lock().unwrap().deref())?; // FIXME: I think we need to decrement the shared leaves // Build metadata space map - - w.flush()?; + let metadata_sm_root = build_metadata_sm(&mut w)?; // Write the superblock - if let Some(xml_sb) = pass.sb { - let sb = superblock::Superblock { - flags: SuperblockFlags { needs_check: false }, - block: SUPERBLOCK_LOCATION, - version: 2, - time: xml_sb.time as u32, - transaction_id: xml_sb.transaction, - metadata_snap: 0, - data_sm_root: vec![0; SPACE_MAP_ROOT_SIZE], - metadata_sm_root: vec![0; SPACE_MAP_ROOT_SIZE], - mapping_root, - details_root, - data_block_size: xml_sb.data_block_size, - nr_metadata_blocks: ctx.engine.get_nr_blocks(), - }; - - write_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?; - } else { - return Err(anyhow!("No superblock found in xml file")); - } + let sb = superblock::Superblock { + flags: SuperblockFlags { needs_check: false }, + block: SUPERBLOCK_LOCATION, + version: 2, + time: pass.sb.time as u32, + transaction_id: pass.sb.transaction, + metadata_snap: 0, + data_sm_root, + metadata_sm_root, + mapping_root, + details_root, + data_block_size: pass.sb.data_block_size, + nr_metadata_blocks: ctx.engine.get_nr_blocks(), + }; + write_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?; Ok(()) } diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index f066032..e9c071c 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -116,12 +116,12 @@ fn pack_superblock(sb: &Superblock, w: &mut W) -> Result<()> { w.write_u32::(sb.time)?; w.write_u64::(sb.transaction_id)?; w.write_u64::(sb.metadata_snap)?; - w.write_all(&[0; SPACE_MAP_ROOT_SIZE])?; // data sm root - w.write_all(&[0; SPACE_MAP_ROOT_SIZE])?; // metadata sm root + w.write_all(&sb.data_sm_root)?; + w.write_all(&sb.metadata_sm_root)?; w.write_u64::(sb.mapping_root)?; w.write_u64::(sb.details_root)?; w.write_u32::(sb.data_block_size)?; - w.write_u32::(BLOCK_SIZE as u32)?; + w.write_u32::((BLOCK_SIZE >> SECTOR_SHIFT) as u32)?; // metadata block size w.write_u64::(sb.nr_metadata_blocks)?; Ok(()) @@ -137,7 +137,7 @@ pub fn write_superblock(engine: &dyn IoEngine, _loc: u64, sb: &Superblock) -> Re } // calculate the checksum - write_checksum(b.get_data(), BT::SUPERBLOCK)?; + write_checksum(b.get_data(), BT::THIN_SUPERBLOCK)?; // write engine.write(&b)?; diff --git a/src/thin/xml.rs b/src/thin/xml.rs index 0cdf7b4..a5c35a9 100644 --- a/src/thin/xml.rs +++ b/src/thin/xml.rs @@ -1,10 +1,11 @@ -use anyhow::{anyhow, Result}; -use std::{borrow::Cow, fmt::Display, io::prelude::*, io::BufReader, io::Write}; +use anyhow::Result; +use std::{io::prelude::*, io::BufReader, io::Write}; -use quick_xml::events::attributes::Attribute; use quick_xml::events::{BytesEnd, BytesStart, Event}; use quick_xml::{Reader, Writer}; +use crate::xml::*; + //--------------------------------------- #[derive(Clone)] @@ -72,18 +73,6 @@ impl XmlWriter { } } -fn mk_attr_<'a, T: Display>(n: T) -> Cow<'a, [u8]> { - let str = format!("{}", n); - Cow::Owned(str.into_bytes()) -} - -fn mk_attr(key: &[u8], value: T) -> Attribute { - Attribute { - key, - value: mk_attr_(value), - } -} - const XML_VERSION: u32 = 2; impl MetadataVisitor for XmlWriter { @@ -187,40 +176,6 @@ impl MetadataVisitor for XmlWriter { //--------------------------------------- -// FIXME: nasty unwraps -fn string_val(kv: &Attribute) -> String { - let v = kv.unescaped_value().unwrap(); - let bytes = v.to_vec(); - String::from_utf8(bytes).unwrap() -} - -// FIXME: there's got to be a way of doing this without copying the string -fn u64_val(kv: &Attribute) -> Result { - let n = string_val(kv).parse::()?; - Ok(n) -} - -fn u32_val(kv: &Attribute) -> Result { - let n = string_val(kv).parse::()?; - Ok(n) -} - -fn bad_attr(_tag: &str, _attr: &[u8]) -> Result { - todo!(); -} - -fn missing_attr(tag: &str, attr: &str) -> Result { - let msg = format!("missing attribute '{}' for tag '{}", attr, tag); - Err(anyhow!(msg)) -} - -fn check_attr(tag: &str, name: &str, maybe_v: Option) -> Result { - match maybe_v { - None => missing_attr(tag, name), - Some(v) => Ok(v), - } -} - fn parse_superblock(e: &BytesStart) -> Result { let mut uuid: Option = None; let mut time: Option = None; diff --git a/src/write_batcher.rs b/src/write_batcher.rs index b911be5..74a04ec 100644 --- a/src/write_batcher.rs +++ b/src/write_batcher.rs @@ -43,6 +43,7 @@ impl WriteBatcher { return Err(anyhow!("out of metadata space")); } + self.allocations.insert(b.unwrap()); Ok(Block::new(b.unwrap())) } diff --git a/src/xml.rs b/src/xml.rs new file mode 100644 index 0000000..577c0e3 --- /dev/null +++ b/src/xml.rs @@ -0,0 +1,63 @@ +use anyhow::anyhow; +use quick_xml::events::attributes::Attribute; +use std::borrow::Cow; +use std::fmt::Display; + +//------------------------------------------ + +pub fn bytes_val<'a>(kv: &'a Attribute) -> Cow<'a, [u8]> { + kv.unescaped_value().unwrap() +} + +// FIXME: nasty unwraps +pub fn string_val(kv: &Attribute) -> String { + let v = kv.unescaped_value().unwrap(); + let bytes = v.to_vec(); + String::from_utf8(bytes).unwrap() +} + +// FIXME: there's got to be a way of doing this without copying the string +pub fn u64_val(kv: &Attribute) -> anyhow::Result { + let n = string_val(kv).parse::()?; + Ok(n) +} + +pub fn u32_val(kv: &Attribute) -> anyhow::Result { + let n = string_val(kv).parse::()?; + Ok(n) +} + +pub fn bool_val(kv: &Attribute) -> anyhow::Result { + let n = string_val(kv).parse::()?; + Ok(n) +} + +pub fn bad_attr(_tag: &str, _attr: &[u8]) -> anyhow::Result { + todo!(); +} + +pub fn check_attr(tag: &str, name: &str, maybe_v: Option) -> anyhow::Result { + match maybe_v { + None => missing_attr(tag, name), + Some(v) => Ok(v), + } +} + +fn missing_attr(tag: &str, attr: &str) -> anyhow::Result { + let msg = format!("missing attribute '{}' for tag '{}", attr, tag); + Err(anyhow!(msg)) +} + +pub fn mk_attr(key: &[u8], value: T) -> Attribute { + Attribute { + key, + value: mk_attr_(value), + } +} + +fn mk_attr_<'a, T: Display>(n: T) -> Cow<'a, [u8]> { + let str = format!("{}", n); + Cow::Owned(str.into_bytes()) +} + +//------------------------------------------ diff --git a/tests/common/cache_xml_generator.rs b/tests/common/cache_xml_generator.rs index f198dea..3621d50 100644 --- a/tests/common/cache_xml_generator.rs +++ b/tests/common/cache_xml_generator.rs @@ -67,7 +67,7 @@ impl XmlGen for CacheGen { v.mappings_b()?; { - let nr_resident = (self.nr_cache_blocks * 100 as u32) / (self.percent_resident as u32); + let nr_resident = (self.nr_cache_blocks * 100u32) / (self.percent_resident as u32); let mut used = HashSet::new(); for n in 0..nr_resident { let mut oblock = 0u64;