Merge branch 'main' into rust-cache-tools
This commit is contained in:
commit
5baeab4a5c
@ -145,11 +145,11 @@ namespace persistent_data {
|
|||||||
{
|
{
|
||||||
internal_node n = spine.get_node<block_traits>();
|
internal_node n = spine.get_node<block_traits>();
|
||||||
|
|
||||||
|
// compact the path if there's only one child
|
||||||
if (n.get_nr_entries() == 1) {
|
if (n.get_nr_entries() == 1) {
|
||||||
block_address b = n.value_at(0);
|
block_address b = n.value_at(0);
|
||||||
read_ref child = tm_.read_lock(b, validator_);
|
read_ref child = tm_.read_lock(b, validator_);
|
||||||
|
|
||||||
// FIXME: is it safe?
|
|
||||||
::memcpy(n.raw(), child.data(), read_ref::BLOCK_SIZE);
|
::memcpy(n.raw(), child.data(), read_ref::BLOCK_SIZE);
|
||||||
|
|
||||||
tm_.get_sm()->dec(child.get_location());
|
tm_.get_sm()->dec(child.get_location());
|
||||||
@ -341,7 +341,6 @@ namespace persistent_data {
|
|||||||
if (nr_left < nr_right) {
|
if (nr_left < nr_right) {
|
||||||
int s = nr_left - target_left;
|
int s = nr_left - target_left;
|
||||||
|
|
||||||
// FIXME: signed & unsigned comparison
|
|
||||||
if (s < 0 && nr_center < static_cast<unsigned>(-s)) {
|
if (s < 0 && nr_center < static_cast<unsigned>(-s)) {
|
||||||
// not enough in central node
|
// not enough in central node
|
||||||
left.move_entries(center, -nr_center);
|
left.move_entries(center, -nr_center);
|
||||||
|
@ -338,7 +338,7 @@ namespace persistent_data {
|
|||||||
unsigned nr_right = rhs.get_nr_entries();
|
unsigned nr_right = rhs.get_nr_entries();
|
||||||
unsigned max_entries = get_max_entries();
|
unsigned max_entries = get_max_entries();
|
||||||
|
|
||||||
if (nr_left - count > max_entries || nr_right - count > max_entries)
|
if (nr_left - count > max_entries || nr_right + count > max_entries)
|
||||||
throw runtime_error("too many entries");
|
throw runtime_error("too many entries");
|
||||||
|
|
||||||
if (count > 0) {
|
if (count > 0) {
|
||||||
|
@ -8,8 +8,9 @@ use std::process;
|
|||||||
use std::process::exit;
|
use std::process::exit;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use thinp::file_utils;
|
use thinp::file_utils;
|
||||||
|
use thinp::io_engine::*;
|
||||||
use thinp::report::*;
|
use thinp::report::*;
|
||||||
use thinp::thin::check::{check, ThinCheckOptions};
|
use thinp::thin::check::{check, ThinCheckOptions, MAX_CONCURRENT_IO};
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let parser = App::new("thin_check")
|
let parser = App::new("thin_check")
|
||||||
@ -97,25 +98,22 @@ fn main() {
|
|||||||
report = Arc::new(mk_simple_report());
|
report = Arc::new(mk_simple_report());
|
||||||
}
|
}
|
||||||
|
|
||||||
if matches.is_present("SYNC_IO") &&
|
if matches.is_present("SYNC_IO") && matches.is_present("ASYNC_IO") {
|
||||||
matches.is_present("ASYNC_IO") {
|
|
||||||
eprintln!("--sync-io and --async-io may not be used at the same time.");
|
eprintln!("--sync-io and --async-io may not be used at the same time.");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut async_io = false;
|
let engine: Arc<dyn IoEngine + Send + Sync>;
|
||||||
if matches.is_present("ASYNC_IO") {
|
|
||||||
async_io = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// redundant since sync is the default.
|
if matches.is_present("ASYNC_IO") {
|
||||||
if matches.is_present("SYNC_IO") {
|
engine = Arc::new(AsyncIoEngine::new(&input_file, MAX_CONCURRENT_IO, false).expect("unable to open input file"));
|
||||||
async_io = false;
|
} else {
|
||||||
|
let nr_threads = std::cmp::max(8, num_cpus::get() * 2);
|
||||||
|
engine = Arc::new(SyncIoEngine::new(&input_file, nr_threads, false).expect("unable to open input file"));
|
||||||
}
|
}
|
||||||
|
|
||||||
let opts = ThinCheckOptions {
|
let opts = ThinCheckOptions {
|
||||||
dev: &input_file,
|
engine: engine,
|
||||||
async_io,
|
|
||||||
sb_only: matches.is_present("SB_ONLY"),
|
sb_only: matches.is_present("SB_ONLY"),
|
||||||
skip_mappings: matches.is_present("SKIP_MAPPINGS"),
|
skip_mappings: matches.is_present("SKIP_MAPPINGS"),
|
||||||
ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"),
|
ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"),
|
||||||
|
@ -27,6 +27,7 @@ fn main() {
|
|||||||
.help("Specify the input xml")
|
.help("Specify the input xml")
|
||||||
.short("i")
|
.short("i")
|
||||||
.long("input")
|
.long("input")
|
||||||
|
.value_name("INPUT")
|
||||||
.required(true),
|
.required(true),
|
||||||
)
|
)
|
||||||
.arg(
|
.arg(
|
||||||
@ -34,6 +35,7 @@ fn main() {
|
|||||||
.help("Specify the output device to check")
|
.help("Specify the output device to check")
|
||||||
.short("o")
|
.short("o")
|
||||||
.long("output")
|
.long("output")
|
||||||
|
.value_name("OUTPUT")
|
||||||
.required(true),
|
.required(true),
|
||||||
)
|
)
|
||||||
.arg(
|
.arg(
|
||||||
|
1
src/cache/check.rs
vendored
1
src/cache/check.rs
vendored
@ -12,6 +12,7 @@ use crate::pdata::array_walker::*;
|
|||||||
use crate::pdata::bitset::*;
|
use crate::pdata::bitset::*;
|
||||||
use crate::pdata::space_map::*;
|
use crate::pdata::space_map::*;
|
||||||
use crate::pdata::space_map_checker::*;
|
use crate::pdata::space_map_checker::*;
|
||||||
|
use crate::pdata::space_map_disk::*;
|
||||||
use crate::pdata::unpack::unpack;
|
use crate::pdata::unpack::unpack;
|
||||||
use crate::report::*;
|
use crate::report::*;
|
||||||
|
|
||||||
|
1
src/cache/hint.rs
vendored
1
src/cache/hint.rs
vendored
@ -1,5 +1,6 @@
|
|||||||
use nom::IResult;
|
use nom::IResult;
|
||||||
use std::convert::TryInto;
|
use std::convert::TryInto;
|
||||||
|
use std::marker::PhantomData;
|
||||||
|
|
||||||
use crate::pdata::unpack::*;
|
use crate::pdata::unpack::*;
|
||||||
|
|
||||||
|
13
src/cache/xml.rs
vendored
13
src/cache/xml.rs
vendored
@ -100,7 +100,8 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn superblock_e(&mut self) -> Result<Visit> {
|
fn superblock_e(&mut self) -> Result<Visit> {
|
||||||
self.w.write_event(Event::End(BytesEnd::borrowed(b"superblock")))?;
|
self.w
|
||||||
|
.write_event(Event::End(BytesEnd::borrowed(b"superblock")))?;
|
||||||
Ok(Visit::Continue)
|
Ok(Visit::Continue)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -112,7 +113,8 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn mappings_e(&mut self) -> Result<Visit> {
|
fn mappings_e(&mut self) -> Result<Visit> {
|
||||||
self.w.write_event(Event::End(BytesEnd::borrowed(b"mappings")))?;
|
self.w
|
||||||
|
.write_event(Event::End(BytesEnd::borrowed(b"mappings")))?;
|
||||||
Ok(Visit::Continue)
|
Ok(Visit::Continue)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -134,7 +136,8 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn hints_e(&mut self) -> Result<Visit> {
|
fn hints_e(&mut self) -> Result<Visit> {
|
||||||
self.w.write_event(Event::End(BytesEnd::borrowed(b"hints")))?;
|
self.w
|
||||||
|
.write_event(Event::End(BytesEnd::borrowed(b"hints")))?;
|
||||||
Ok(Visit::Continue)
|
Ok(Visit::Continue)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -155,7 +158,8 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn discards_e(&mut self) -> Result<Visit> {
|
fn discards_e(&mut self) -> Result<Visit> {
|
||||||
self.w.write_event(Event::End(BytesEnd::borrowed(b"discards")))?;
|
self.w
|
||||||
|
.write_event(Event::End(BytesEnd::borrowed(b"discards")))?;
|
||||||
Ok(Visit::Continue)
|
Ok(Visit::Continue)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -172,4 +176,3 @@ impl<W: Write> MetadataVisitor for XmlWriter<W> {
|
|||||||
Ok(Visit::Continue)
|
Ok(Visit::Continue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6,7 +6,6 @@ use std::io::Cursor;
|
|||||||
|
|
||||||
const BLOCK_SIZE: u64 = 4096;
|
const BLOCK_SIZE: u64 = 4096;
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
const MAGIC: u64 = 0xa537a0aa6309ef77;
|
|
||||||
const SUPERBLOCK_CSUM_XOR: u32 = 160774;
|
const SUPERBLOCK_CSUM_XOR: u32 = 160774;
|
||||||
const BITMAP_CSUM_XOR: u32 = 240779;
|
const BITMAP_CSUM_XOR: u32 = 240779;
|
||||||
const INDEX_CSUM_XOR: u32 = 160478;
|
const INDEX_CSUM_XOR: u32 = 160478;
|
||||||
|
@ -26,6 +26,8 @@ pub struct Block {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Block {
|
impl Block {
|
||||||
|
// Creates a new block that corresponds to the given location. The
|
||||||
|
// memory is not initialised.
|
||||||
pub fn new(loc: u64) -> Block {
|
pub fn new(loc: u64) -> Block {
|
||||||
let layout = Layout::from_size_align(BLOCK_SIZE, ALIGN).unwrap();
|
let layout = Layout::from_size_align(BLOCK_SIZE, ALIGN).unwrap();
|
||||||
let ptr = unsafe { alloc(layout) };
|
let ptr = unsafe { alloc(layout) };
|
||||||
@ -42,6 +44,12 @@ impl Block {
|
|||||||
pub fn get_data<'a>(&self) -> &'a mut [u8] {
|
pub fn get_data<'a>(&self) -> &'a mut [u8] {
|
||||||
unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, BLOCK_SIZE) }
|
unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, BLOCK_SIZE) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn zero(&mut self) {
|
||||||
|
unsafe {
|
||||||
|
std::ptr::write_bytes(self.data, 0, BLOCK_SIZE);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for Block {
|
impl Drop for Block {
|
||||||
|
@ -19,6 +19,7 @@ pub mod cache;
|
|||||||
pub mod checksum;
|
pub mod checksum;
|
||||||
pub mod file_utils;
|
pub mod file_utils;
|
||||||
pub mod io_engine;
|
pub mod io_engine;
|
||||||
|
pub mod math;
|
||||||
pub mod pack;
|
pub mod pack;
|
||||||
pub mod pdata;
|
pub mod pdata;
|
||||||
pub mod report;
|
pub mod report;
|
||||||
|
7
src/math.rs
Normal file
7
src/math.rs
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
pub fn div_up(v: usize, divisor: usize) -> usize {
|
||||||
|
v / divisor + (v % divisor != 0) as usize
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn div_down(v: usize, divisor: usize) -> usize {
|
||||||
|
v / divisor
|
||||||
|
}
|
@ -31,10 +31,7 @@ pub fn to_delta(ns: &[u64]) -> Vec<Delta> {
|
|||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
count -= 1;
|
count -= 1;
|
||||||
ds.push(Neg {
|
ds.push(Neg { delta, count });
|
||||||
delta,
|
|
||||||
count,
|
|
||||||
});
|
|
||||||
base -= delta * count;
|
base -= delta * count;
|
||||||
}
|
}
|
||||||
Equal => {
|
Equal => {
|
||||||
@ -54,10 +51,7 @@ pub fn to_delta(ns: &[u64]) -> Vec<Delta> {
|
|||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
count -= 1;
|
count -= 1;
|
||||||
ds.push(Pos {
|
ds.push(Pos { delta, count });
|
||||||
delta,
|
|
||||||
count,
|
|
||||||
});
|
|
||||||
base += delta * count;
|
base += delta * count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -539,6 +539,7 @@ pub fn unpack_node<V: Unpack>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !is_root {
|
if !is_root {
|
||||||
|
/*
|
||||||
let min = header.max_entries / 3;
|
let min = header.max_entries / 3;
|
||||||
if header.nr_entries < min {
|
if header.nr_entries < min {
|
||||||
return Err(node_err_s(
|
return Err(node_err_s(
|
||||||
@ -549,6 +550,7 @@ pub fn unpack_node<V: Unpack>(
|
|||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -558,7 +560,7 @@ pub fn unpack_node<V: Unpack>(
|
|||||||
for k in &keys {
|
for k in &keys {
|
||||||
if let Some(l) = last {
|
if let Some(l) = last {
|
||||||
if k <= l {
|
if k <= l {
|
||||||
return Err(node_err(&path, "keys out of order"));
|
return Err(node_err(&path, &format!("keys out of order: {} <= {}", k, l)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -13,6 +13,70 @@ use crate::write_batcher::*;
|
|||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
/// A little ref counter abstraction. Used to manage counts for btree
|
||||||
|
/// values (eg, the block/time in a thin mapping tree).
|
||||||
|
pub trait RefCounter<Value> {
|
||||||
|
fn get(&self, v: &Value) -> Result<u32>;
|
||||||
|
fn inc(&mut self, v: &Value) -> Result<()>;
|
||||||
|
fn dec(&mut self, v: &Value) -> Result<()>;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NoopRC {}
|
||||||
|
|
||||||
|
impl<Value> RefCounter<Value> for NoopRC {
|
||||||
|
fn get(&self, _v: &Value) -> Result<u32> {
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
fn inc(&mut self, _v: &Value) -> Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn dec(&mut self, _v: &Value) -> Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wraps a space map up to become a RefCounter.
|
||||||
|
struct SMRefCounter {
|
||||||
|
sm: Arc<Mutex<dyn SpaceMap>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RefCounter<u64> for SMRefCounter {
|
||||||
|
fn get(&self, v: &u64) -> Result<u32> {
|
||||||
|
self.sm.lock().unwrap().get(*v)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn inc(&mut self, v: &u64) -> Result<()> {
|
||||||
|
self.sm.lock().unwrap().inc(*v, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dec(&mut self, v: &u64) -> Result<()> {
|
||||||
|
self.sm.lock().unwrap().dec(*v)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
|
||||||
|
// Building a btree for a given set of values is straight forward.
|
||||||
|
// But often we want to merge shared subtrees into the btree we're
|
||||||
|
// building, which _is_ complicated. Requiring rebalancing of nodes,
|
||||||
|
// and careful copy-on-write operations so we don't disturb the shared
|
||||||
|
// subtree.
|
||||||
|
//
|
||||||
|
// To avoid these problems this code never produces shared internal nodes.
|
||||||
|
// With the large fan out of btrees this isn't really a problem; we'll
|
||||||
|
// allocate more nodes than optimum, but not many compared to the number
|
||||||
|
// of leaves. Also we can pack the leaves much better than the kernel
|
||||||
|
// does due to out of order insertions.
|
||||||
|
//
|
||||||
|
// There are thus two stages to building a btree.
|
||||||
|
//
|
||||||
|
// i) Produce a list of populated leaves. These leaves may well be shared.
|
||||||
|
// ii) Build the upper levels of the btree above the leaves.
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
|
||||||
|
/// Pack the given node ready to write to disk.
|
||||||
pub fn pack_node<W: WriteBytesExt, V: Pack + Unpack>(node: &Node<V>, w: &mut W) -> Result<()> {
|
pub fn pack_node<W: WriteBytesExt, V: Pack + Unpack>(node: &Node<V>, w: &mut W) -> Result<()> {
|
||||||
match node {
|
match node {
|
||||||
Node::Internal {
|
Node::Internal {
|
||||||
@ -62,115 +126,43 @@ pub fn pack_node<W: WriteBytesExt, V: Pack + Unpack>(node: &Node<V>, w: &mut W)
|
|||||||
|
|
||||||
pub fn calc_max_entries<V: Unpack>() -> usize {
|
pub fn calc_max_entries<V: Unpack>() -> usize {
|
||||||
let elt_size = 8 + V::disk_size() as usize;
|
let elt_size = 8 + V::disk_size() as usize;
|
||||||
((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize
|
let total = ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize;
|
||||||
|
total / 3 * 3
|
||||||
}
|
}
|
||||||
|
|
||||||
//------------------------------------------
|
pub struct WriteResult {
|
||||||
|
first_key: u64,
|
||||||
struct Entries<V> {
|
loc: u64,
|
||||||
pub max_entries: usize,
|
|
||||||
entries: VecDeque<(u64, V)>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
enum Action<V> {
|
/// Write a node to a free metadata block.
|
||||||
EmitNode(Vec<u64>, Vec<V>), // keys, values
|
fn write_node_<V: Unpack + Pack>(w: &mut WriteBatcher, mut node: Node<V>) -> Result<WriteResult> {
|
||||||
}
|
|
||||||
|
|
||||||
use Action::*;
|
|
||||||
|
|
||||||
impl<V> Entries<V> {
|
|
||||||
pub fn new(max_entries: usize) -> Entries<V> {
|
|
||||||
Entries {
|
|
||||||
max_entries,
|
|
||||||
entries: VecDeque::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn add_entry(&mut self, k: u64, v: V) -> Vec<Action<V>> {
|
|
||||||
let mut result = Vec::new();
|
|
||||||
|
|
||||||
if self.full() {
|
|
||||||
let (keys, values) = self.pop(self.max_entries);
|
|
||||||
result.push(EmitNode(keys, values));
|
|
||||||
}
|
|
||||||
|
|
||||||
self.entries.push_back((k, v));
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
||||||
fn complete_(&mut self, result: &mut Vec<Action<V>>) {
|
|
||||||
let n = self.entries.len();
|
|
||||||
|
|
||||||
if n >= self.max_entries {
|
|
||||||
let n1 = n / 2;
|
|
||||||
let n2 = n - n1;
|
|
||||||
let (keys1, values1) = self.pop(n1);
|
|
||||||
let (keys2, values2) = self.pop(n2);
|
|
||||||
|
|
||||||
result.push(EmitNode(keys1, values1));
|
|
||||||
result.push(EmitNode(keys2, values2));
|
|
||||||
} else if n > 0 {
|
|
||||||
let (keys, values) = self.pop(n);
|
|
||||||
result.push(EmitNode(keys, values));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn complete(&mut self) -> Vec<Action<V>> {
|
|
||||||
let mut result = Vec::new();
|
|
||||||
self.complete_(&mut result);
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
||||||
fn full(&self) -> bool {
|
|
||||||
self.entries.len() >= 2 * self.max_entries
|
|
||||||
}
|
|
||||||
|
|
||||||
fn pop(&mut self, count: usize) -> (Vec<u64>, Vec<V>) {
|
|
||||||
let mut keys = Vec::new();
|
|
||||||
let mut values = Vec::new();
|
|
||||||
|
|
||||||
for _i in 0..count {
|
|
||||||
let (k, v) = self.entries.pop_front().unwrap();
|
|
||||||
keys.push(k);
|
|
||||||
values.push(v);
|
|
||||||
}
|
|
||||||
|
|
||||||
(keys, values)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//------------------------------------------
|
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct NodeSummary {
|
|
||||||
block: u64,
|
|
||||||
nr_entries: usize,
|
|
||||||
key_low: u64,
|
|
||||||
key_high: u64, // inclusive
|
|
||||||
}
|
|
||||||
|
|
||||||
//------------------------------------------
|
|
||||||
|
|
||||||
fn write_node_<V: Unpack + Pack>(w: &mut WriteBatcher, mut node: Node<V>) -> Result<(u64, u64)> {
|
|
||||||
let keys = node.get_keys();
|
let keys = node.get_keys();
|
||||||
let first_key = *keys.first().unwrap_or(&0u64);
|
let first_key = keys.first().unwrap_or(&0u64).clone();
|
||||||
|
|
||||||
let loc = w.alloc()?;
|
let b = w.alloc()?;
|
||||||
node.set_block(loc);
|
node.set_block(b.loc);
|
||||||
|
|
||||||
let b = Block::new(loc);
|
|
||||||
let mut cursor = Cursor::new(b.get_data());
|
let mut cursor = Cursor::new(b.get_data());
|
||||||
pack_node(&node, &mut cursor)?;
|
pack_node(&node, &mut cursor)?;
|
||||||
|
let loc = b.loc;
|
||||||
w.write(b, checksum::BT::NODE)?;
|
w.write(b, checksum::BT::NODE)?;
|
||||||
|
|
||||||
Ok((first_key, loc))
|
Ok(WriteResult { first_key, loc })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_leaf<V: Unpack + Pack>(
|
/// A node writer takes a Vec of values and packs them into
|
||||||
w: &mut WriteBatcher,
|
/// a btree node. It's up to the specific implementation to
|
||||||
keys: Vec<u64>,
|
/// decide if it produces internal or leaf nodes.
|
||||||
values: Vec<V>,
|
pub trait NodeIO<V: Unpack + Pack> {
|
||||||
) -> Result<(u64, u64)> {
|
fn write(&self, w: &mut WriteBatcher, keys: Vec<u64>, values: Vec<V>) -> Result<WriteResult>;
|
||||||
|
fn read(&self, w: &mut WriteBatcher, block: u64) -> Result<(Vec<u64>, Vec<V>)>;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct LeafIO {}
|
||||||
|
|
||||||
|
impl<V: Unpack + Pack> NodeIO<V> for LeafIO {
|
||||||
|
fn write(&self, w: &mut WriteBatcher, keys: Vec<u64>, values: Vec<V>) -> Result<WriteResult> {
|
||||||
let header = NodeHeader {
|
let header = NodeHeader {
|
||||||
block: 0,
|
block: 0,
|
||||||
is_leaf: true,
|
is_leaf: true,
|
||||||
@ -186,9 +178,24 @@ fn write_leaf<V: Unpack + Pack>(
|
|||||||
};
|
};
|
||||||
|
|
||||||
write_node_(w, node)
|
write_node_(w, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read(&self, w: &mut WriteBatcher, block: u64) -> Result<(Vec<u64>, Vec<V>)> {
|
||||||
|
let b = w.read(block)?;
|
||||||
|
let path = Vec::new();
|
||||||
|
match unpack_node::<V>(&path, b.get_data(), true, true)? {
|
||||||
|
Node::Internal { .. } => {
|
||||||
|
panic!("unexpected internal node");
|
||||||
|
}
|
||||||
|
Node::Leaf { keys, values, .. } => Ok((keys, values)),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_internal(w: &mut WriteBatcher, keys: Vec<u64>, values: Vec<u64>) -> Result<(u64, u64)> {
|
struct InternalIO {}
|
||||||
|
|
||||||
|
impl NodeIO<u64> for InternalIO {
|
||||||
|
fn write(&self, w: &mut WriteBatcher, keys: Vec<u64>, values: Vec<u64>) -> Result<WriteResult> {
|
||||||
let header = NodeHeader {
|
let header = NodeHeader {
|
||||||
block: 0,
|
block: 0,
|
||||||
is_leaf: false,
|
is_leaf: false,
|
||||||
@ -204,114 +211,281 @@ fn write_internal(w: &mut WriteBatcher, keys: Vec<u64>, values: Vec<u64>) -> Res
|
|||||||
};
|
};
|
||||||
|
|
||||||
write_node_(w, node)
|
write_node_(w, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read(&self, w: &mut WriteBatcher, block: u64) -> Result<(Vec<u64>, Vec<u64>)> {
|
||||||
|
let b = w.read(block)?;
|
||||||
|
let path = Vec::new();
|
||||||
|
match unpack_node::<u64>(&path, b.get_data(), true, true)? {
|
||||||
|
Node::Internal { keys, values, .. } => Ok((keys, values)),
|
||||||
|
Node::Leaf { .. } => {
|
||||||
|
panic!("unexpected leaf node");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Builder<V: Unpack + Pack> {
|
//------------------------------------------
|
||||||
w: WriteBatcher,
|
|
||||||
entries: Entries<V>,
|
|
||||||
|
|
||||||
max_internal_entries: usize,
|
/// This takes a sequence of values or nodes, and builds a vector of leaf nodes.
|
||||||
internal_entries: Vec<Entries<u64>>,
|
/// Care is taken to make sure that all nodes are at least half full unless there's
|
||||||
|
/// only a single node.
|
||||||
root: u64,
|
pub struct NodeBuilder<V: Pack + Unpack> {
|
||||||
|
nio: Box<dyn NodeIO<V>>,
|
||||||
|
value_rc: Box<dyn RefCounter<V>>,
|
||||||
|
max_entries_per_node: usize,
|
||||||
|
values: VecDeque<(u64, V)>,
|
||||||
|
nodes: Vec<NodeSummary>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<V: Unpack + Pack> Builder<V> {
|
/// When the builder is including pre-built nodes it has to decide whether
|
||||||
pub fn new(
|
/// to use the node as given, or read it and import the values directly
|
||||||
engine: Arc<dyn IoEngine + Send + Sync>,
|
/// for balancing reasons. This struct is used to stop us re-reading
|
||||||
sm: Arc<Mutex<dyn SpaceMap>>,
|
/// the NodeHeaders of nodes that are shared multiple times.
|
||||||
) -> Builder<V> {
|
#[derive(Clone)]
|
||||||
let max_entries = calc_max_entries::<V>();
|
pub struct NodeSummary {
|
||||||
let max_internal_entries = calc_max_entries::<u64>();
|
block: u64,
|
||||||
|
key: u64,
|
||||||
|
nr_entries: usize,
|
||||||
|
|
||||||
Builder {
|
/// This node was passed in pre-built. Important for deciding if
|
||||||
w: WriteBatcher::new(engine, sm, 256),
|
/// we need to adjust the ref counts if we unpack.
|
||||||
entries: Entries::new(max_entries),
|
shared: bool,
|
||||||
max_internal_entries,
|
}
|
||||||
internal_entries: Vec::new(),
|
|
||||||
root: 0,
|
impl<'a, V: Pack + Unpack + Clone> NodeBuilder<V> {
|
||||||
|
/// Create a new NodeBuilder
|
||||||
|
pub fn new(nio: Box<dyn NodeIO<V>>, value_rc: Box<dyn RefCounter<V>>) -> Self {
|
||||||
|
NodeBuilder {
|
||||||
|
nio,
|
||||||
|
value_rc,
|
||||||
|
max_entries_per_node: calc_max_entries::<V>(),
|
||||||
|
values: VecDeque::new(),
|
||||||
|
nodes: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/// Push a single value. This may emit a new node, hence the Result
|
||||||
pub fn add_entry(&mut self, k: u64, v: V) -> Result<()> {
|
/// return type. The value's ref count will be incremented.
|
||||||
let actions = self.entries.add_entry(k, v);
|
pub fn push_value(&mut self, w: &mut WriteBatcher, key: u64, val: V) -> Result<()> {
|
||||||
for a in actions {
|
// Have we got enough values to emit a node? We try and keep
|
||||||
self.perform_action(a)?;
|
// at least max_entries_per_node entries unflushed so we
|
||||||
|
// can ensure the final node is balanced properly.
|
||||||
|
if self.values.len() == self.max_entries_per_node * 2 {
|
||||||
|
self.emit_node(w)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.value_rc.inc(&val)?;
|
||||||
|
self.values.push_back((key, val));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn add_leaf_node(&mut self, leaf: &NodeSummary) -> Result<()> {
|
/// Push a number of prebuilt, shared nodes. The builder may decide to not
|
||||||
match leaf.nr_entries {
|
/// use a shared node, instead reading the values and packing them
|
||||||
n if n == 0 => {
|
/// directly. This may do IO to emit nodes, so returns a Result.
|
||||||
// Do nothing
|
/// Any shared nodes that are used have their block incremented in
|
||||||
},
|
/// the space map. Will only increment the ref count for values
|
||||||
n if n < (self.entries.max_entries / 2) => {
|
/// contained in the nodes if it unpacks them.
|
||||||
// FIXME: what if we've already queued a handful of entries for a node?
|
pub fn push_nodes(&mut self, w: &mut WriteBatcher, nodes: &Vec<NodeSummary>) -> Result<()> {
|
||||||
// Add the entries individually
|
assert!(nodes.len() > 0);
|
||||||
todo!();
|
|
||||||
},
|
// As a sanity check we make sure that all the shared nodes contain the
|
||||||
_n => {
|
// minimum nr of entries.
|
||||||
let actions = self.entries.complete();
|
let half_full = self.max_entries_per_node / 2;
|
||||||
for a in actions {
|
for n in nodes {
|
||||||
self.perform_action(a)?;
|
if n.nr_entries < half_full {
|
||||||
|
panic!("under populated node");
|
||||||
}
|
}
|
||||||
self.add_internal_entry(0, leaf.key_low, leaf.block)?;
|
}
|
||||||
|
|
||||||
|
// Decide if we're going to use the pre-built nodes.
|
||||||
|
if (self.values.len() > 0) && (self.values.len() < half_full) {
|
||||||
|
// To avoid writing an under populated node we have to grab some
|
||||||
|
// values from the first of the shared nodes.
|
||||||
|
let (keys, values) = self.read_node(w, nodes.get(0).unwrap().block)?;
|
||||||
|
|
||||||
|
for i in 0..keys.len() {
|
||||||
|
self.value_rc.inc(&values[i])?;
|
||||||
|
self.values.push_back((keys[i], values[i].clone()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush all the values.
|
||||||
|
self.emit_all(w)?;
|
||||||
|
|
||||||
|
// Add the remaining nodes.
|
||||||
|
for i in 1..nodes.len() {
|
||||||
|
let n = nodes.get(i).unwrap();
|
||||||
|
w.sm.lock().unwrap().inc(n.block, 1)?;
|
||||||
|
self.nodes.push(n.clone());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Flush all the values.
|
||||||
|
self.emit_all(w)?;
|
||||||
|
|
||||||
|
// add the nodes
|
||||||
|
for n in nodes {
|
||||||
|
w.sm.lock().unwrap().inc(n.block, 1)?;
|
||||||
|
self.nodes.push(n.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn complete(mut self) -> Result<u64> {
|
/// Signal that no more values or nodes will be pushed. Returns a
|
||||||
let actions = self.entries.complete();
|
/// vector of the built nodes. Consumes the builder.
|
||||||
for a in actions {
|
pub fn complete(mut self, w: &mut WriteBatcher) -> Result<Vec<NodeSummary>> {
|
||||||
self.perform_action(a)?;
|
let half_full = self.max_entries_per_node / 2;
|
||||||
}
|
|
||||||
self.w.flush()?;
|
if (self.values.len() > 0) && (self.values.len() < half_full) && (self.nodes.len() > 0) {
|
||||||
Ok(self.root)
|
// We don't have enough values to emit a node. So we're going to
|
||||||
|
// have to rebalance with the previous node.
|
||||||
|
self.unshift_node(w)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
//--------------------
|
self.emit_all(w)?;
|
||||||
|
|
||||||
fn add_internal_entry(&mut self, level: usize, k: u64, v: u64) -> Result<()> {
|
if self.nodes.len() == 0 {
|
||||||
if self.internal_entries.len() == level {
|
self.emit_empty_leaf(w)?
|
||||||
self.internal_entries
|
|
||||||
.push(Entries::new(self.max_internal_entries));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let actions = self.internal_entries[level].add_entry(k, v);
|
Ok(self.nodes)
|
||||||
|
|
||||||
for a in actions {
|
|
||||||
self.perform_internal_action(level, a)?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//-------------------------
|
||||||
|
|
||||||
|
// We're only interested in the keys and values from the node, and
|
||||||
|
// not whether it's a leaf or internal node.
|
||||||
|
fn read_node(&self, w: &mut WriteBatcher, block: u64) -> Result<(Vec<u64>, Vec<V>)> {
|
||||||
|
self.nio.read(w, block)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a node with the first 'nr_entries' values.
|
||||||
|
fn emit_values(&mut self, w: &mut WriteBatcher, nr_entries: usize) -> Result<()> {
|
||||||
|
assert!(nr_entries <= self.values.len());
|
||||||
|
|
||||||
|
// Write the node
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
let mut values = Vec::new();
|
||||||
|
|
||||||
|
for _i in 0..nr_entries {
|
||||||
|
let (k, v) = self.values.pop_front().unwrap();
|
||||||
|
keys.push(k);
|
||||||
|
values.push(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
let wresult = self.nio.write(w, keys, values)?;
|
||||||
|
|
||||||
|
// Push a summary to the 'nodes' vector.
|
||||||
|
self.nodes.push(NodeSummary {
|
||||||
|
block: wresult.loc,
|
||||||
|
key: wresult.first_key,
|
||||||
|
nr_entries,
|
||||||
|
shared: false,
|
||||||
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn perform_internal_action(&mut self, level: usize, action: Action<u64>) -> Result<()> {
|
/// Writes a full node.
|
||||||
match action {
|
fn emit_node(&mut self, w: &mut WriteBatcher) -> Result<()> {
|
||||||
EmitNode(keys, values) => {
|
self.emit_values(w, self.max_entries_per_node)
|
||||||
let (k, loc) = write_internal(&mut self.w, keys, values)?;
|
|
||||||
self.add_internal_entry(level + 1, k, loc)?;
|
|
||||||
self.root = loc;
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Emits all remaining values. Panics if there are more than 2 *
|
||||||
|
/// max_entries_per_node values.
|
||||||
|
fn emit_all(&mut self, w: &mut WriteBatcher) -> Result<()> {
|
||||||
|
match self.values.len() {
|
||||||
|
0 => {
|
||||||
|
// There's nothing to emit
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
n if n <= self.max_entries_per_node => {
|
||||||
fn perform_action<V2: Unpack + Pack>(&mut self, action: Action<V2>) -> Result<()> {
|
// Emit a single node.
|
||||||
match action {
|
self.emit_values(w, n)
|
||||||
EmitNode(keys, values) => {
|
|
||||||
let (k, loc) = write_leaf(&mut self.w, keys, values)?;
|
|
||||||
self.add_internal_entry(0, k, loc)?;
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
n if n <= self.max_entries_per_node * 2 => {
|
||||||
|
// Emit two nodes.
|
||||||
|
let n1 = n / 2;
|
||||||
|
let n2 = n - n1;
|
||||||
|
self.emit_values(w, n1)?;
|
||||||
|
self.emit_values(w, n2)
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
panic!("self.values shouldn't have more than 2 * max_entries_per_node entries");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emit_empty_leaf(&mut self, w: &mut WriteBatcher) -> Result<()> {
|
||||||
|
self.emit_values(w, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pops the last node, and prepends it's values to 'self.values'. Used
|
||||||
|
/// to rebalance when we have insufficient values for a final node. The
|
||||||
|
/// node is decremented in the space map.
|
||||||
|
fn unshift_node(&mut self, w: &mut WriteBatcher) -> Result<()> {
|
||||||
|
let ls = self.nodes.pop().unwrap();
|
||||||
|
let (keys, values) = self.read_node(w, ls.block)?;
|
||||||
|
w.sm.lock().unwrap().dec(ls.block)?;
|
||||||
|
|
||||||
|
let mut vals = VecDeque::new();
|
||||||
|
|
||||||
|
for i in 0..keys.len() {
|
||||||
|
// We only need to inc the values if the node was pre built.
|
||||||
|
if ls.shared {
|
||||||
|
self.value_rc.inc(&values[i])?;
|
||||||
|
}
|
||||||
|
vals.push_back((keys[i], values[i].clone()));
|
||||||
|
}
|
||||||
|
|
||||||
|
vals.append(&mut self.values);
|
||||||
|
std::mem::swap(&mut self.values, &mut vals);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
pub struct Builder<V: Unpack + Pack> {
|
||||||
|
leaf_builder: NodeBuilder<V>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<V: Unpack + Pack + Clone> Builder<V> {
|
||||||
|
pub fn new(value_rc: Box<dyn RefCounter<V>>) -> Builder<V> {
|
||||||
|
Builder {
|
||||||
|
leaf_builder: NodeBuilder::new(Box::new(LeafIO {}), value_rc),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn push_value(&mut self, w: &mut WriteBatcher, k: u64, v: V) -> Result<()> {
|
||||||
|
self.leaf_builder.push_value(w, k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn push_leaves(&mut self, w: &mut WriteBatcher, leaves: &Vec<NodeSummary>) -> Result<()> {
|
||||||
|
self.leaf_builder.push_nodes(w, leaves)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn complete(self, w: &mut WriteBatcher) -> Result<u64> {
|
||||||
|
let mut nodes = self.leaf_builder.complete(w)?;
|
||||||
|
|
||||||
|
// Now we iterate, adding layers of internal nodes until we end
|
||||||
|
// up with a single root.
|
||||||
|
while nodes.len() > 1 {
|
||||||
|
let mut builder = NodeBuilder::new(
|
||||||
|
Box::new(InternalIO {}),
|
||||||
|
Box::new(SMRefCounter { sm: w.sm.clone() }),
|
||||||
|
);
|
||||||
|
|
||||||
|
for n in nodes {
|
||||||
|
builder.push_value(w, n.key, n.block)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes = builder.complete(w)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(nodes.len() == 1);
|
||||||
|
Ok(nodes[0].block)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
@ -3,10 +3,10 @@ pub mod array_walker;
|
|||||||
pub mod bitset;
|
pub mod bitset;
|
||||||
pub mod btree;
|
pub mod btree;
|
||||||
pub mod btree_builder;
|
pub mod btree_builder;
|
||||||
pub mod btree_merge;
|
|
||||||
pub mod btree_leaf_walker;
|
pub mod btree_leaf_walker;
|
||||||
|
pub mod btree_merge;
|
||||||
pub mod btree_walker;
|
pub mod btree_walker;
|
||||||
pub mod space_map;
|
pub mod space_map;
|
||||||
pub mod space_map_checker;
|
pub mod space_map_checker;
|
||||||
|
pub mod space_map_disk;
|
||||||
pub mod unpack;
|
pub mod unpack;
|
||||||
|
|
||||||
|
@ -1,223 +1,7 @@
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::Result;
|
||||||
use byteorder::{LittleEndian, WriteBytesExt};
|
|
||||||
use fixedbitset::FixedBitSet;
|
use fixedbitset::FixedBitSet;
|
||||||
use nom::{multi::count, number::complete::*, IResult};
|
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
use std::boxed::Box;
|
use std::boxed::Box;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
use crate::io_engine::*;
|
|
||||||
use crate::pdata::unpack::{Pack, Unpack};
|
|
||||||
|
|
||||||
//------------------------------------------
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct SMRoot {
|
|
||||||
pub nr_blocks: u64,
|
|
||||||
pub nr_allocated: u64,
|
|
||||||
pub bitmap_root: u64,
|
|
||||||
pub ref_count_root: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn unpack_root(data: &[u8]) -> Result<SMRoot> {
|
|
||||||
match SMRoot::unpack(data) {
|
|
||||||
Err(_e) => Err(anyhow!("couldn't parse SMRoot")),
|
|
||||||
Ok((_i, v)) => Ok(v),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Unpack for SMRoot {
|
|
||||||
fn disk_size() -> u32 {
|
|
||||||
32
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unpack(data: &[u8]) -> IResult<&[u8], SMRoot> {
|
|
||||||
let (i, nr_blocks) = le_u64(data)?;
|
|
||||||
let (i, nr_allocated) = le_u64(i)?;
|
|
||||||
let (i, bitmap_root) = le_u64(i)?;
|
|
||||||
let (i, ref_count_root) = le_u64(i)?;
|
|
||||||
|
|
||||||
Ok((
|
|
||||||
i,
|
|
||||||
SMRoot {
|
|
||||||
nr_blocks,
|
|
||||||
nr_allocated,
|
|
||||||
bitmap_root,
|
|
||||||
ref_count_root,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//------------------------------------------
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
|
||||||
pub struct IndexEntry {
|
|
||||||
pub blocknr: u64,
|
|
||||||
pub nr_free: u32,
|
|
||||||
pub none_free_before: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Unpack for IndexEntry {
|
|
||||||
fn disk_size() -> u32 {
|
|
||||||
16
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
|
|
||||||
let (i, blocknr) = le_u64(data)?;
|
|
||||||
let (i, nr_free) = le_u32(i)?;
|
|
||||||
let (i, none_free_before) = le_u32(i)?;
|
|
||||||
|
|
||||||
Ok((
|
|
||||||
i,
|
|
||||||
IndexEntry {
|
|
||||||
blocknr,
|
|
||||||
nr_free,
|
|
||||||
none_free_before,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//------------------------------------------
|
|
||||||
|
|
||||||
pub const MAX_METADATA_BITMAPS: usize = 255;
|
|
||||||
|
|
||||||
pub struct MetadataIndex {
|
|
||||||
pub indexes: Vec<IndexEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Unpack for MetadataIndex {
|
|
||||||
fn disk_size() -> u32 {
|
|
||||||
BLOCK_SIZE as u32
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
|
|
||||||
let (i, _csum) = le_u32(data)?;
|
|
||||||
let (i, _padding) = le_u32(i)?;
|
|
||||||
let (i, _blocknr) = le_u64(i)?;
|
|
||||||
let (i, indexes) = count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?;
|
|
||||||
|
|
||||||
Ok((i, MetadataIndex { indexes }))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//------------------------------------------
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct BitmapHeader {
|
|
||||||
pub csum: u32,
|
|
||||||
pub not_used: u32,
|
|
||||||
pub blocknr: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Unpack for BitmapHeader {
|
|
||||||
fn disk_size() -> u32 {
|
|
||||||
16
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
|
|
||||||
let (i, csum) = le_u32(data)?;
|
|
||||||
let (i, not_used) = le_u32(i)?;
|
|
||||||
let (i, blocknr) = le_u64(i)?;
|
|
||||||
|
|
||||||
Ok((
|
|
||||||
i,
|
|
||||||
BitmapHeader {
|
|
||||||
csum,
|
|
||||||
not_used,
|
|
||||||
blocknr,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Pack for BitmapHeader {
|
|
||||||
fn pack<W: WriteBytesExt>(&self, out: &mut W) -> Result<()> {
|
|
||||||
out.write_u32::<LittleEndian>(self.csum)?;
|
|
||||||
out.write_u32::<LittleEndian>(self.not_used)?;
|
|
||||||
out.write_u64::<LittleEndian>(self.blocknr)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq)]
|
|
||||||
pub enum BitmapEntry {
|
|
||||||
Small(u8),
|
|
||||||
Overflow,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct Bitmap {
|
|
||||||
pub header: BitmapHeader,
|
|
||||||
pub entries: Vec<BitmapEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Unpack for Bitmap {
|
|
||||||
fn disk_size() -> u32 {
|
|
||||||
BLOCK_SIZE as u32
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
|
|
||||||
let (mut i, header) = BitmapHeader::unpack(data)?;
|
|
||||||
|
|
||||||
let nr_words = (BLOCK_SIZE - BitmapHeader::disk_size() as usize) / 8;
|
|
||||||
let mut entries = Vec::with_capacity(nr_words * 32);
|
|
||||||
for _w in 0..nr_words {
|
|
||||||
let (tmp, mut word) = le_u64(i)?;
|
|
||||||
|
|
||||||
for _b in 0..32 {
|
|
||||||
let val = word & 0x3;
|
|
||||||
word >>= 2;
|
|
||||||
|
|
||||||
// The bits are stored with the high bit at b * 2 + 1,
|
|
||||||
// and low at b *2. So we have to interpret this val.
|
|
||||||
entries.push(match val {
|
|
||||||
0 => BitmapEntry::Small(0),
|
|
||||||
1 => BitmapEntry::Small(2),
|
|
||||||
2 => BitmapEntry::Small(1),
|
|
||||||
_ => BitmapEntry::Overflow,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
i = tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok((i, Bitmap { header, entries }))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Pack for Bitmap {
|
|
||||||
fn pack<W: WriteBytesExt>(&self, out: &mut W) -> Result<()> {
|
|
||||||
use BitmapEntry::*;
|
|
||||||
BitmapHeader::pack(&self.header, out)?;
|
|
||||||
|
|
||||||
for chunk in self.entries.chunks(32) {
|
|
||||||
let mut w = 0u64;
|
|
||||||
for e in chunk {
|
|
||||||
w >>= 2;
|
|
||||||
match e {
|
|
||||||
Small(0) => {}
|
|
||||||
Small(1) => {
|
|
||||||
w |= 0x2 << 62;
|
|
||||||
}
|
|
||||||
Small(2) => {
|
|
||||||
w |= 0x1 << 62;
|
|
||||||
}
|
|
||||||
Small(_) => {
|
|
||||||
return Err(anyhow!("Bad small value in bitmap entry"));
|
|
||||||
}
|
|
||||||
Overflow => {
|
|
||||||
w |= 0x3 << 62;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
u64::pack(&w, out)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -226,13 +10,22 @@ pub trait SpaceMap {
|
|||||||
fn get_nr_allocated(&self) -> Result<u64>;
|
fn get_nr_allocated(&self) -> Result<u64>;
|
||||||
fn get(&self, b: u64) -> Result<u32>;
|
fn get(&self, b: u64) -> Result<u32>;
|
||||||
|
|
||||||
// Returns the old ref count
|
/// Returns the old ref count
|
||||||
fn set(&mut self, b: u64, v: u32) -> Result<u32>;
|
fn set(&mut self, b: u64, v: u32) -> Result<u32>;
|
||||||
|
|
||||||
fn inc(&mut self, begin: u64, len: u64) -> Result<()>;
|
fn inc(&mut self, begin: u64, len: u64) -> Result<()>;
|
||||||
|
|
||||||
// Finds a block with a zero reference count. Increments the
|
/// Returns true if the block is now free
|
||||||
// count.
|
fn dec(&mut self, b: u64) -> Result<bool> {
|
||||||
|
let old = self.get(b)?;
|
||||||
|
assert!(old > 0);
|
||||||
|
self.set(b, old - 1)?;
|
||||||
|
|
||||||
|
Ok(old == 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finds a block with a zero reference count. Increments the
|
||||||
|
/// count.
|
||||||
fn alloc(&mut self) -> Result<Option<u64>>;
|
fn alloc(&mut self) -> Result<Option<u64>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -7,6 +7,7 @@ use crate::io_engine::IoEngine;
|
|||||||
use crate::pdata::btree::{self, *};
|
use crate::pdata::btree::{self, *};
|
||||||
use crate::pdata::btree_walker::*;
|
use crate::pdata::btree_walker::*;
|
||||||
use crate::pdata::space_map::*;
|
use crate::pdata::space_map::*;
|
||||||
|
use crate::pdata::space_map_disk::*;
|
||||||
use crate::pdata::unpack::*;
|
use crate::pdata::unpack::*;
|
||||||
use crate::report::Report;
|
use crate::report::Report;
|
||||||
|
|
||||||
|
399
src/pdata/space_map_disk.rs
Normal file
399
src/pdata/space_map_disk.rs
Normal file
@ -0,0 +1,399 @@
|
|||||||
|
use anyhow::{anyhow, Result};
|
||||||
|
use byteorder::{LittleEndian, WriteBytesExt};
|
||||||
|
use nom::{number::complete::*, IResult};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
use crate::checksum;
|
||||||
|
use crate::io_engine::*;
|
||||||
|
use crate::math::*;
|
||||||
|
use crate::pdata::btree_builder::*;
|
||||||
|
use crate::pdata::space_map::*;
|
||||||
|
use crate::pdata::unpack::*;
|
||||||
|
use crate::write_batcher::*;
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
const MAX_METADATA_BITMAPS: usize = 255;
|
||||||
|
// const MAX_METADATA_BLOCKS: u64 = 255 * ((1 << 14) - 64);
|
||||||
|
const ENTRIES_PER_BYTE: usize = 4;
|
||||||
|
const ENTRIES_PER_BITMAP: usize = WORDS_PER_BITMAP * 8 * ENTRIES_PER_BYTE;
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
pub struct IndexEntry {
|
||||||
|
pub blocknr: u64,
|
||||||
|
pub nr_free: u32,
|
||||||
|
pub none_free_before: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Unpack for IndexEntry {
|
||||||
|
fn disk_size() -> u32 {
|
||||||
|
16
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unpack(i: &[u8]) -> IResult<&[u8], IndexEntry> {
|
||||||
|
let (i, blocknr) = le_u64(i)?;
|
||||||
|
let (i, nr_free) = le_u32(i)?;
|
||||||
|
let (i, none_free_before) = le_u32(i)?;
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
i,
|
||||||
|
IndexEntry {
|
||||||
|
blocknr,
|
||||||
|
nr_free,
|
||||||
|
none_free_before,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Pack for IndexEntry {
|
||||||
|
fn pack<W: WriteBytesExt>(&self, w: &mut W) -> Result<()> {
|
||||||
|
w.write_u64::<LittleEndian>(self.blocknr)?;
|
||||||
|
w.write_u32::<LittleEndian>(self.nr_free)?;
|
||||||
|
w.write_u32::<LittleEndian>(self.none_free_before)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
pub struct MetadataIndex {
|
||||||
|
pub blocknr: u64,
|
||||||
|
pub indexes: Vec<IndexEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Unpack for MetadataIndex {
|
||||||
|
fn disk_size() -> u32 {
|
||||||
|
BLOCK_SIZE as u32
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unpack(i: &[u8]) -> IResult<&[u8], MetadataIndex> {
|
||||||
|
// FIXME: check the checksum
|
||||||
|
let (i, _csum) = le_u32(i)?;
|
||||||
|
let (i, _padding) = le_u32(i)?;
|
||||||
|
let (i, blocknr) = le_u64(i)?;
|
||||||
|
let (i, indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?;
|
||||||
|
|
||||||
|
Ok((i, MetadataIndex { blocknr, indexes }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Pack for MetadataIndex {
|
||||||
|
fn pack<W: WriteBytesExt>(&self, w: &mut W) -> Result<()> {
|
||||||
|
w.write_u32::<LittleEndian>(0)?; // csum
|
||||||
|
w.write_u32::<LittleEndian>(0)?; // padding
|
||||||
|
w.write_u64::<LittleEndian>(self.blocknr)?;
|
||||||
|
|
||||||
|
assert!(self.indexes.len() <= MAX_METADATA_BITMAPS);
|
||||||
|
|
||||||
|
for ie in &self.indexes {
|
||||||
|
ie.pack(w)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
const WORDS_PER_BITMAP: usize = (BLOCK_SIZE - 16) / 8;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum BitmapEntry {
|
||||||
|
Small(u8),
|
||||||
|
Overflow,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Bitmap {
|
||||||
|
pub blocknr: u64,
|
||||||
|
pub entries: Vec<BitmapEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Unpack for Bitmap {
|
||||||
|
fn disk_size() -> u32 {
|
||||||
|
BLOCK_SIZE as u32
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unpack(data: &[u8]) -> IResult<&[u8], Self> {
|
||||||
|
let (i, _csum) = le_u32(data)?;
|
||||||
|
let (i, _not_used) = le_u32(i)?;
|
||||||
|
let (mut i, blocknr) = le_u64(i)?;
|
||||||
|
|
||||||
|
let header_size = 16;
|
||||||
|
let nr_words = (BLOCK_SIZE - header_size) / 8;
|
||||||
|
let mut entries = Vec::with_capacity(nr_words * 32);
|
||||||
|
for _w in 0..nr_words {
|
||||||
|
let (tmp, mut word) = le_u64(i)?;
|
||||||
|
|
||||||
|
for _b in 0..32 {
|
||||||
|
let val = word & 0x3;
|
||||||
|
word >>= 2;
|
||||||
|
|
||||||
|
// The bits are stored with the high bit at b * 2 + 1,
|
||||||
|
// and low at b *2. So we have to interpret this val.
|
||||||
|
entries.push(match val {
|
||||||
|
0 => BitmapEntry::Small(0),
|
||||||
|
1 => BitmapEntry::Small(2),
|
||||||
|
2 => BitmapEntry::Small(1),
|
||||||
|
_ => BitmapEntry::Overflow,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
i = tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok((i, Bitmap { blocknr, entries }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Pack for Bitmap {
|
||||||
|
fn pack<W: WriteBytesExt>(&self, out: &mut W) -> Result<()> {
|
||||||
|
use BitmapEntry::*;
|
||||||
|
|
||||||
|
out.write_u32::<LittleEndian>(0)?;
|
||||||
|
out.write_u32::<LittleEndian>(0)?;
|
||||||
|
out.write_u64::<LittleEndian>(self.blocknr)?;
|
||||||
|
|
||||||
|
for chunk in self.entries.chunks(32) {
|
||||||
|
let mut w = 0u64;
|
||||||
|
for e in chunk {
|
||||||
|
w >>= 2;
|
||||||
|
match e {
|
||||||
|
Small(0) => {}
|
||||||
|
Small(1) => {
|
||||||
|
w |= 0x2 << 62;
|
||||||
|
}
|
||||||
|
Small(2) => {
|
||||||
|
w |= 0x1 << 62;
|
||||||
|
}
|
||||||
|
Small(_) => {
|
||||||
|
return Err(anyhow!("Bad small value in bitmap entry"));
|
||||||
|
}
|
||||||
|
Overflow => {
|
||||||
|
w |= 0x3 << 62;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
u64::pack(&w, out)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct SMRoot {
|
||||||
|
pub nr_blocks: u64,
|
||||||
|
pub nr_allocated: u64,
|
||||||
|
pub bitmap_root: u64,
|
||||||
|
pub ref_count_root: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Unpack for SMRoot {
|
||||||
|
fn disk_size() -> u32 {
|
||||||
|
32
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unpack(i: &[u8]) -> IResult<&[u8], Self> {
|
||||||
|
let (i, nr_blocks) = le_u64(i)?;
|
||||||
|
let (i, nr_allocated) = le_u64(i)?;
|
||||||
|
let (i, bitmap_root) = le_u64(i)?;
|
||||||
|
let (i, ref_count_root) = le_u64(i)?;
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
i,
|
||||||
|
SMRoot {
|
||||||
|
nr_blocks,
|
||||||
|
nr_allocated,
|
||||||
|
bitmap_root,
|
||||||
|
ref_count_root,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn unpack_root(data: &[u8]) -> Result<SMRoot> {
|
||||||
|
match SMRoot::unpack(data) {
|
||||||
|
Err(_e) => Err(anyhow!("couldn't parse SMRoot")),
|
||||||
|
Ok((_i, v)) => Ok(v),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Pack for SMRoot {
|
||||||
|
fn pack<W: WriteBytesExt>(&self, w: &mut W) -> Result<()> {
|
||||||
|
w.write_u64::<LittleEndian>(self.nr_blocks)?;
|
||||||
|
w.write_u64::<LittleEndian>(self.nr_allocated)?;
|
||||||
|
w.write_u64::<LittleEndian>(self.bitmap_root)?;
|
||||||
|
w.write_u64::<LittleEndian>(self.ref_count_root)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
pub fn write_common(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result<(Vec<IndexEntry>, u64)> {
|
||||||
|
use BitmapEntry::*;
|
||||||
|
|
||||||
|
let mut index_entries = Vec::new();
|
||||||
|
let mut overflow_builder: Builder<u32> = Builder::new(Box::new(NoopRC {}));
|
||||||
|
|
||||||
|
// how many bitmaps do we need?
|
||||||
|
for bm in 0..div_up(sm.get_nr_blocks()? as usize, ENTRIES_PER_BITMAP) {
|
||||||
|
let mut entries = Vec::with_capacity(ENTRIES_PER_BITMAP);
|
||||||
|
let mut first_free: Option<u32> = None;
|
||||||
|
let mut nr_free: u32 = 0;
|
||||||
|
for i in 0..ENTRIES_PER_BITMAP {
|
||||||
|
let b: u64 = ((bm * ENTRIES_PER_BITMAP) as u64) + i as u64;
|
||||||
|
if b > sm.get_nr_blocks()? {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let rc = sm.get(b)?;
|
||||||
|
let e = match rc {
|
||||||
|
0 => {
|
||||||
|
nr_free += 1;
|
||||||
|
if first_free.is_none() {
|
||||||
|
first_free = Some(i as u32);
|
||||||
|
}
|
||||||
|
Small(0)
|
||||||
|
}
|
||||||
|
1 => Small(1),
|
||||||
|
2 => Small(2),
|
||||||
|
_ => {
|
||||||
|
overflow_builder.push_value(w, b as u64, rc)?;
|
||||||
|
Overflow
|
||||||
|
}
|
||||||
|
};
|
||||||
|
entries.push(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// allocate a new block
|
||||||
|
let b = w.alloc()?;
|
||||||
|
let mut cursor = Cursor::new(b.get_data());
|
||||||
|
|
||||||
|
// write the bitmap to it
|
||||||
|
let blocknr = b.loc;
|
||||||
|
let bitmap = Bitmap { blocknr, entries };
|
||||||
|
bitmap.pack(&mut cursor)?;
|
||||||
|
w.write(b, checksum::BT::BITMAP)?;
|
||||||
|
|
||||||
|
// Insert into the index tree
|
||||||
|
let ie = IndexEntry {
|
||||||
|
blocknr,
|
||||||
|
nr_free,
|
||||||
|
none_free_before: first_free.unwrap_or(ENTRIES_PER_BITMAP as u32),
|
||||||
|
};
|
||||||
|
index_entries.push(ie);
|
||||||
|
}
|
||||||
|
|
||||||
|
let ref_count_root = overflow_builder.complete(w)?;
|
||||||
|
Ok((index_entries, ref_count_root))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn write_disk_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result<SMRoot> {
|
||||||
|
let (index_entries, ref_count_root) = write_common(w, sm)?;
|
||||||
|
|
||||||
|
let mut index_builder: Builder<IndexEntry> = Builder::new(Box::new(NoopRC {}));
|
||||||
|
for (i, ie) in index_entries.iter().enumerate() {
|
||||||
|
index_builder.push_value(w, i as u64, *ie)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let bitmap_root = index_builder.complete(w)?;
|
||||||
|
|
||||||
|
Ok(SMRoot {
|
||||||
|
nr_blocks: sm.get_nr_blocks()?,
|
||||||
|
nr_allocated: sm.get_nr_allocated()?,
|
||||||
|
bitmap_root,
|
||||||
|
ref_count_root,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
//----------------------------
|
||||||
|
|
||||||
|
fn block_to_bitmap(b: u64) -> usize {
|
||||||
|
(b / ENTRIES_PER_BITMAP as u64) as usize
|
||||||
|
}
|
||||||
|
|
||||||
|
fn adjust_counts(w: &mut WriteBatcher, ie: &IndexEntry, allocs: &[u64]) -> Result<IndexEntry> {
|
||||||
|
use BitmapEntry::*;
|
||||||
|
|
||||||
|
let mut first_free = ie.none_free_before;
|
||||||
|
let mut nr_free = ie.nr_free - allocs.len() as u32;
|
||||||
|
|
||||||
|
// Read the bitmap
|
||||||
|
let bitmap_block = w.engine.read(ie.blocknr)?;
|
||||||
|
let (_, mut bitmap) = Bitmap::unpack(bitmap_block.get_data())?;
|
||||||
|
|
||||||
|
// Update all the entries
|
||||||
|
for a in allocs {
|
||||||
|
if first_free == *a as u32 {
|
||||||
|
first_free = *a as u32 + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if bitmap.entries[*a as usize] == Small(0) {
|
||||||
|
nr_free -= 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
bitmap.entries[*a as usize] = Small(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write the bitmap
|
||||||
|
let mut cur = Cursor::new(bitmap_block.get_data());
|
||||||
|
bitmap.pack(&mut cur)?;
|
||||||
|
w.write(bitmap_block, checksum::BT::BITMAP)?;
|
||||||
|
|
||||||
|
// Return the adjusted index entry
|
||||||
|
Ok(IndexEntry {
|
||||||
|
blocknr: ie.blocknr,
|
||||||
|
nr_free,
|
||||||
|
none_free_before: first_free,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn write_metadata_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result<SMRoot> {
|
||||||
|
w.clear_allocations();
|
||||||
|
let (mut indexes, ref_count_root) = write_common(w, sm)?;
|
||||||
|
|
||||||
|
let bitmap_root = w.alloc()?;
|
||||||
|
|
||||||
|
// Now we need to patch up the counts for the metadata that was used for storing
|
||||||
|
// the space map itself. These ref counts all went from 0 to 1.
|
||||||
|
let allocations = w.clear_allocations();
|
||||||
|
|
||||||
|
// Sort the allocations by bitmap
|
||||||
|
let mut by_bitmap = BTreeMap::new();
|
||||||
|
for b in allocations {
|
||||||
|
let bitmap = block_to_bitmap(b);
|
||||||
|
(*by_bitmap.entry(bitmap).or_insert(Vec::new())).push(b % ENTRIES_PER_BITMAP as u64);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (bitmap, allocs) in by_bitmap {
|
||||||
|
indexes[bitmap] = adjust_counts(w, &indexes[bitmap], &allocs)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write out the metadata index
|
||||||
|
let metadata_index = MetadataIndex {
|
||||||
|
blocknr: bitmap_root.loc,
|
||||||
|
indexes,
|
||||||
|
};
|
||||||
|
let mut cur = Cursor::new(bitmap_root.get_data());
|
||||||
|
metadata_index.pack(&mut cur)?;
|
||||||
|
let loc = bitmap_root.loc;
|
||||||
|
w.write(bitmap_root, checksum::BT::INDEX)?;
|
||||||
|
|
||||||
|
Ok(SMRoot {
|
||||||
|
nr_blocks: sm.get_nr_blocks()?,
|
||||||
|
nr_allocated: sm.get_nr_allocated()?,
|
||||||
|
bitmap_root: loc,
|
||||||
|
ref_count_root,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------
|
@ -1,6 +1,6 @@
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use nom::{number::complete::*, IResult};
|
|
||||||
use byteorder::{LittleEndian, WriteBytesExt};
|
use byteorder::{LittleEndian, WriteBytesExt};
|
||||||
|
use nom::{number::complete::*, IResult};
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
@ -29,7 +29,7 @@ pub struct Report {
|
|||||||
inner: Mutex<Box<dyn ReportInner + Send>>,
|
inner: Mutex<Box<dyn ReportInner + Send>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
trait ReportInner {
|
pub trait ReportInner {
|
||||||
fn set_title(&mut self, txt: &str);
|
fn set_title(&mut self, txt: &str);
|
||||||
fn set_sub_title(&mut self, txt: &str);
|
fn set_sub_title(&mut self, txt: &str);
|
||||||
fn progress(&mut self, percent: u8);
|
fn progress(&mut self, percent: u8);
|
||||||
@ -38,7 +38,7 @@ trait ReportInner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Report {
|
impl Report {
|
||||||
fn new(inner: Box<dyn ReportInner + Send>) -> Report {
|
pub fn new(inner: Box<dyn ReportInner + Send>) -> Report {
|
||||||
Report {
|
Report {
|
||||||
outcome: Mutex::new(Success),
|
outcome: Mutex::new(Success),
|
||||||
inner: Mutex::new(inner),
|
inner: Mutex::new(inner),
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
|
use std::io::{Read, Seek, SeekFrom, Write};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::io::{Seek, SeekFrom, Write, Read};
|
|
||||||
//use std::os::unix::fs::OpenOptionsExt;
|
//use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
|
||||||
pub type Sector = u64;
|
pub type Sector = u64;
|
||||||
@ -13,7 +13,6 @@ pub struct Region {
|
|||||||
pub len: Sector,
|
pub len: Sector,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn copy_step<W>(file: &mut W, src_byte: u64, dest_byte: u64, len: usize) -> Result<()>
|
fn copy_step<W>(file: &mut W, src_byte: u64, dest_byte: u64, len: usize) -> Result<()>
|
||||||
where
|
where
|
||||||
W: Write + Seek + Read,
|
W: Write + Seek + Read,
|
||||||
@ -38,7 +37,12 @@ where
|
|||||||
let mut written = 0;
|
let mut written = 0;
|
||||||
while written != len_bytes {
|
while written != len_bytes {
|
||||||
let step = u64::min(len_bytes - written, MAX_BYTES);
|
let step = u64::min(len_bytes - written, MAX_BYTES);
|
||||||
copy_step(file, src_bytes + written, dest_bytes + written, step as usize)?;
|
copy_step(
|
||||||
|
file,
|
||||||
|
src_bytes + written,
|
||||||
|
dest_bytes + written,
|
||||||
|
step as usize,
|
||||||
|
)?;
|
||||||
written += step;
|
written += step;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -1,3 +1,5 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use byteorder::WriteBytesExt;
|
||||||
use nom::{number::complete::*, IResult};
|
use nom::{number::complete::*, IResult};
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
@ -31,6 +33,13 @@ impl Unpack for BlockTime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Pack for BlockTime {
|
||||||
|
fn pack<W: WriteBytesExt>(&self, data: &mut W) -> Result<()> {
|
||||||
|
let bt: u64 = (self.block << 24) | self.time as u64;
|
||||||
|
bt.pack(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl fmt::Display for BlockTime {
|
impl fmt::Display for BlockTime {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
write!(f, "{} @ {}", self.block, self.time)
|
write!(f, "{} @ {}", self.block, self.time)
|
||||||
|
@ -11,6 +11,7 @@ use crate::pdata::btree::{self, *};
|
|||||||
use crate::pdata::btree_walker::*;
|
use crate::pdata::btree_walker::*;
|
||||||
use crate::pdata::space_map::*;
|
use crate::pdata::space_map::*;
|
||||||
use crate::pdata::space_map_checker::*;
|
use crate::pdata::space_map_checker::*;
|
||||||
|
use crate::pdata::space_map_disk::*;
|
||||||
use crate::pdata::unpack::*;
|
use crate::pdata::unpack::*;
|
||||||
use crate::report::*;
|
use crate::report::*;
|
||||||
use crate::thin::block_time::*;
|
use crate::thin::block_time::*;
|
||||||
@ -79,11 +80,10 @@ fn inc_superblock(sm: &ASpaceMap) -> Result<()> {
|
|||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
const MAX_CONCURRENT_IO: u32 = 1024;
|
pub const MAX_CONCURRENT_IO: u32 = 1024;
|
||||||
|
|
||||||
pub struct ThinCheckOptions<'a> {
|
pub struct ThinCheckOptions {
|
||||||
pub dev: &'a Path,
|
pub engine: Arc<dyn IoEngine + Send + Sync>,
|
||||||
pub async_io: bool,
|
|
||||||
pub sb_only: bool,
|
pub sb_only: bool,
|
||||||
pub skip_mappings: bool,
|
pub skip_mappings: bool,
|
||||||
pub ignore_non_fatal: bool,
|
pub ignore_non_fatal: bool,
|
||||||
@ -194,25 +194,12 @@ fn check_mapping_bottom_level(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mk_context(opts: &ThinCheckOptions) -> Result<Context> {
|
fn mk_context(engine: Arc<dyn IoEngine + Send + Sync>, report: Arc<Report>) -> Result<Context> {
|
||||||
let engine: Arc<dyn IoEngine + Send + Sync>;
|
let nr_threads = std::cmp::max(8, num_cpus::get() * 2);
|
||||||
let nr_threads;
|
|
||||||
|
|
||||||
if opts.async_io {
|
|
||||||
nr_threads = std::cmp::min(4, num_cpus::get());
|
|
||||||
engine = Arc::new(AsyncIoEngine::new(
|
|
||||||
opts.dev,
|
|
||||||
MAX_CONCURRENT_IO,
|
|
||||||
opts.auto_repair,
|
|
||||||
)?);
|
|
||||||
} else {
|
|
||||||
nr_threads = std::cmp::max(8, num_cpus::get() * 2);
|
|
||||||
engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads, opts.auto_repair)?);
|
|
||||||
}
|
|
||||||
let pool = ThreadPool::new(nr_threads);
|
let pool = ThreadPool::new(nr_threads);
|
||||||
|
|
||||||
Ok(Context {
|
Ok(Context {
|
||||||
report: opts.report.clone(),
|
report,
|
||||||
engine,
|
engine,
|
||||||
pool,
|
pool,
|
||||||
})
|
})
|
||||||
@ -231,7 +218,7 @@ fn bail_out(ctx: &Context, task: &str) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn check(opts: ThinCheckOptions) -> Result<()> {
|
pub fn check(opts: ThinCheckOptions) -> Result<()> {
|
||||||
let ctx = mk_context(&opts)?;
|
let ctx = mk_context(opts.engine.clone(), opts.report.clone())?;
|
||||||
|
|
||||||
// FIXME: temporarily get these out
|
// FIXME: temporarily get these out
|
||||||
let report = &ctx.report;
|
let report = &ctx.report;
|
||||||
@ -355,3 +342,107 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
// Some callers wish to know which blocks are allocated.
|
||||||
|
pub struct CheckMaps {
|
||||||
|
pub metadata_sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
|
||||||
|
pub data_sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_with_maps(engine: Arc<dyn IoEngine + Send + Sync>, report: Arc<Report>) -> Result<CheckMaps> {
|
||||||
|
let ctx = mk_context(engine.clone(), report.clone())?;
|
||||||
|
report.set_title("Checking thin metadata");
|
||||||
|
|
||||||
|
// superblock
|
||||||
|
let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?;
|
||||||
|
|
||||||
|
report.info(&format!("TRANSACTION_ID={}", sb.transaction_id));
|
||||||
|
|
||||||
|
let metadata_root = unpack::<SMRoot>(&sb.metadata_sm_root[0..])?;
|
||||||
|
let mut path = Vec::new();
|
||||||
|
path.push(0);
|
||||||
|
|
||||||
|
// Device details. We read this once to get the number of thin devices, and hence the
|
||||||
|
// maximum metadata ref count. Then create metadata space map, and reread to increment
|
||||||
|
// the ref counts for that metadata.
|
||||||
|
let devs = btree_to_map::<DeviceDetail>(
|
||||||
|
&mut path,
|
||||||
|
engine.clone(),
|
||||||
|
false,
|
||||||
|
sb.details_root,
|
||||||
|
)?;
|
||||||
|
let nr_devs = devs.len();
|
||||||
|
let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32);
|
||||||
|
inc_superblock(&metadata_sm)?;
|
||||||
|
|
||||||
|
report.set_sub_title("device details tree");
|
||||||
|
let _devs = btree_to_map_with_sm::<DeviceDetail>(
|
||||||
|
&mut path,
|
||||||
|
engine.clone(),
|
||||||
|
metadata_sm.clone(),
|
||||||
|
false,
|
||||||
|
sb.details_root,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let (tid, stop_progress) = spawn_progress_thread(
|
||||||
|
metadata_sm.clone(),
|
||||||
|
metadata_root.nr_allocated,
|
||||||
|
report.clone(),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// mapping top level
|
||||||
|
report.set_sub_title("mapping tree");
|
||||||
|
let roots = btree_to_map_with_path::<u64>(
|
||||||
|
&mut path,
|
||||||
|
engine.clone(),
|
||||||
|
metadata_sm.clone(),
|
||||||
|
false,
|
||||||
|
sb.mapping_root,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// mapping bottom level
|
||||||
|
let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
|
||||||
|
let data_sm = core_sm(root.nr_blocks, nr_devs as u32);
|
||||||
|
check_mapping_bottom_level(&ctx, &metadata_sm, &data_sm, &roots)?;
|
||||||
|
bail_out(&ctx, "mapping tree")?;
|
||||||
|
|
||||||
|
//-----------------------------------------
|
||||||
|
|
||||||
|
report.set_sub_title("data space map");
|
||||||
|
let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
|
||||||
|
let _data_leaks = check_disk_space_map(
|
||||||
|
engine.clone(),
|
||||||
|
report.clone(),
|
||||||
|
root,
|
||||||
|
data_sm.clone(),
|
||||||
|
metadata_sm.clone(),
|
||||||
|
false,
|
||||||
|
)?;
|
||||||
|
bail_out(&ctx, "data space map")?;
|
||||||
|
|
||||||
|
//-----------------------------------------
|
||||||
|
|
||||||
|
report.set_sub_title("metadata space map");
|
||||||
|
let root = unpack::<SMRoot>(&sb.metadata_sm_root[0..])?;
|
||||||
|
report.info(&format!(
|
||||||
|
"METADATA_FREE_BLOCKS={}",
|
||||||
|
root.nr_blocks - root.nr_allocated
|
||||||
|
));
|
||||||
|
|
||||||
|
// Now the counts should be correct and we can check it.
|
||||||
|
let _metadata_leaks =
|
||||||
|
check_metadata_space_map(engine.clone(), report, root, metadata_sm.clone(), false)?;
|
||||||
|
bail_out(&ctx, "metadata space map")?;
|
||||||
|
|
||||||
|
//-----------------------------------------
|
||||||
|
|
||||||
|
stop_progress.store(true, Ordering::Relaxed);
|
||||||
|
tid.join().unwrap();
|
||||||
|
|
||||||
|
Ok(CheckMaps {
|
||||||
|
metadata_sm: metadata_sm.clone(),
|
||||||
|
data_sm: data_sm.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
@ -1,7 +1,9 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use byteorder::{LittleEndian, WriteBytesExt};
|
||||||
|
use nom::{number::complete::*, IResult};
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
use crate::pdata::unpack::*;
|
use crate::pdata::unpack::*;
|
||||||
use nom::{number::complete::*, IResult};
|
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -15,11 +17,11 @@ pub struct DeviceDetail {
|
|||||||
|
|
||||||
impl fmt::Display for DeviceDetail {
|
impl fmt::Display for DeviceDetail {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
write!(f, "mapped = {}, trans = {}, create = {}, snap = {}",
|
write!(
|
||||||
self.mapped_blocks,
|
f,
|
||||||
self.transaction_id,
|
"mapped = {}, trans = {}, create = {}, snap = {}",
|
||||||
self.creation_time,
|
self.mapped_blocks, self.transaction_id, self.creation_time, self.snapshotted_time
|
||||||
self.snapshotted_time)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -47,4 +49,14 @@ impl Unpack for DeviceDetail {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Pack for DeviceDetail {
|
||||||
|
fn pack<W: WriteBytesExt>(&self, w: &mut W) -> Result<()> {
|
||||||
|
w.write_u64::<LittleEndian>(self.mapped_blocks)?;
|
||||||
|
w.write_u64::<LittleEndian>(self.transaction_id)?;
|
||||||
|
w.write_u32::<LittleEndian>(self.creation_time)?;
|
||||||
|
w.write_u32::<LittleEndian>(self.snapshotted_time)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
@ -11,6 +11,7 @@ use crate::pdata::btree::{self, *};
|
|||||||
use crate::pdata::btree_leaf_walker::*;
|
use crate::pdata::btree_leaf_walker::*;
|
||||||
use crate::pdata::btree_walker::*;
|
use crate::pdata::btree_walker::*;
|
||||||
use crate::pdata::space_map::*;
|
use crate::pdata::space_map::*;
|
||||||
|
use crate::pdata::space_map_disk::*;
|
||||||
use crate::pdata::unpack::*;
|
use crate::pdata::unpack::*;
|
||||||
use crate::report::*;
|
use crate::report::*;
|
||||||
use crate::thin::block_time::*;
|
use crate::thin::block_time::*;
|
||||||
@ -287,7 +288,7 @@ fn find_shared_nodes(
|
|||||||
|
|
||||||
// We have to get the leaves so w is consumed and the &mut on sm
|
// We have to get the leaves so w is consumed and the &mut on sm
|
||||||
// is dropped.
|
// is dropped.
|
||||||
let leaves = w.get_leaves();
|
let _leaves = w.get_leaves();
|
||||||
let mut shared = BTreeSet::new();
|
let mut shared = BTreeSet::new();
|
||||||
{
|
{
|
||||||
for i in 0..sm.get_nr_blocks().unwrap() {
|
for i in 0..sm.get_nr_blocks().unwrap() {
|
||||||
@ -297,6 +298,8 @@ fn find_shared_nodes(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
// FIXME: why?!!
|
||||||
// we're not interested in leaves (roots will get re-added later).
|
// we're not interested in leaves (roots will get re-added later).
|
||||||
{
|
{
|
||||||
for i in 0..leaves.len() {
|
for i in 0..leaves.len() {
|
||||||
@ -305,6 +308,7 @@ fn find_shared_nodes(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
Ok((shared, sm))
|
Ok((shared, sm))
|
||||||
}
|
}
|
||||||
@ -616,9 +620,11 @@ pub fn dump(opts: ThinDumpOptions) -> Result<()> {
|
|||||||
let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?;
|
let sb = read_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION)?;
|
||||||
let md = build_metadata(&ctx, &sb)?;
|
let md = build_metadata(&ctx, &sb)?;
|
||||||
|
|
||||||
|
/*
|
||||||
ctx.report
|
ctx.report
|
||||||
.set_title("Optimising metadata to improve leaf packing");
|
.set_title("Optimising metadata to improve leaf packing");
|
||||||
let md = optimise_metadata(md)?;
|
let md = optimise_metadata(md)?;
|
||||||
|
*/
|
||||||
dump_metadata(&ctx, &sb, &md)
|
dump_metadata(&ctx, &sb, &md)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,8 +1,203 @@
|
|||||||
use anyhow::Result;
|
use anyhow::{anyhow, Result};
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fs::OpenOptions;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::io_engine::*;
|
||||||
|
use crate::pdata::btree_builder::*;
|
||||||
|
use crate::pdata::space_map::*;
|
||||||
use crate::report::*;
|
use crate::report::*;
|
||||||
|
use crate::thin::block_time::*;
|
||||||
|
use crate::thin::device_detail::*;
|
||||||
|
use crate::thin::superblock::{self, *};
|
||||||
|
use crate::thin::xml::{self, *};
|
||||||
|
use crate::write_batcher::*;
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
|
||||||
|
enum MappedSection {
|
||||||
|
Def(String),
|
||||||
|
Dev(u32),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for MappedSection {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
MappedSection::Def(name) => write!(f, "Def {}", name),
|
||||||
|
MappedSection::Dev(thin_id) => write!(f, "Device {}", thin_id),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Pass1Result {
|
||||||
|
sb: Option<xml::Superblock>,
|
||||||
|
devices: BTreeMap<u32, (DeviceDetail, Vec<NodeSummary>)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Pass1<'a> {
|
||||||
|
w: &'a mut WriteBatcher,
|
||||||
|
|
||||||
|
current_dev: Option<DeviceDetail>,
|
||||||
|
sub_trees: BTreeMap<String, Vec<NodeSummary>>,
|
||||||
|
|
||||||
|
// The builder for the current shared sub tree or device
|
||||||
|
map: Option<(MappedSection, NodeBuilder<BlockTime>)>,
|
||||||
|
|
||||||
|
result: Pass1Result,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Pass1<'a> {
|
||||||
|
fn new(w: &'a mut WriteBatcher) -> Self {
|
||||||
|
Pass1 {
|
||||||
|
w,
|
||||||
|
current_dev: None,
|
||||||
|
sub_trees: BTreeMap::new(),
|
||||||
|
map: None,
|
||||||
|
result: Pass1Result {
|
||||||
|
sb: None,
|
||||||
|
devices: BTreeMap::new(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_result(self) -> Pass1Result {
|
||||||
|
self.result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn begin_section(&mut self, section: MappedSection) -> Result<Visit> {
|
||||||
|
if let Some((outer, _)) = self.map.as_ref() {
|
||||||
|
let msg = format!(
|
||||||
|
"Nested subtrees are not allowed '{}' within '{}'",
|
||||||
|
section, outer
|
||||||
|
);
|
||||||
|
return Err(anyhow!(msg));
|
||||||
|
}
|
||||||
|
|
||||||
|
let value_rc = Box::new(NoopRC {});
|
||||||
|
let leaf_builder = NodeBuilder::new(Box::new(LeafIO {}), value_rc);
|
||||||
|
|
||||||
|
self.map = Some((section, leaf_builder));
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn end_section(&mut self) -> Result<(MappedSection, Vec<NodeSummary>)> {
|
||||||
|
let mut current = None;
|
||||||
|
std::mem::swap(&mut self.map, &mut current);
|
||||||
|
|
||||||
|
if let Some((name, nodes)) = current {
|
||||||
|
Ok((name, nodes.complete(self.w)?))
|
||||||
|
} else {
|
||||||
|
let msg = format!("Unbalanced </def> tag");
|
||||||
|
Err(anyhow!(msg))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> MetadataVisitor for Pass1<'a> {
|
||||||
|
fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
|
||||||
|
self.result.sb = Some(sb.clone());
|
||||||
|
self.w.alloc()?;
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn superblock_e(&mut self) -> Result<Visit> {
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn def_shared_b(&mut self, name: &str) -> Result<Visit> {
|
||||||
|
self.begin_section(MappedSection::Def(name.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn def_shared_e(&mut self) -> Result<Visit> {
|
||||||
|
if let (MappedSection::Def(name), nodes) = self.end_section()? {
|
||||||
|
self.sub_trees.insert(name, nodes);
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("unexpected </def>"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn device_b(&mut self, d: &Device) -> Result<Visit> {
|
||||||
|
self.current_dev = Some(DeviceDetail {
|
||||||
|
mapped_blocks: d.mapped_blocks,
|
||||||
|
transaction_id: d.transaction,
|
||||||
|
creation_time: d.creation_time as u32,
|
||||||
|
snapshotted_time: d.snap_time as u32,
|
||||||
|
});
|
||||||
|
self.begin_section(MappedSection::Dev(d.dev_id))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn device_e(&mut self) -> Result<Visit> {
|
||||||
|
if let Some(detail) = self.current_dev.take() {
|
||||||
|
if let (MappedSection::Dev(thin_id), nodes) = self.end_section()? {
|
||||||
|
self.result.devices.insert(thin_id, (detail, nodes));
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("internal error, couldn't find device details"))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("unexpected </device>"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map(&mut self, m: &Map) -> Result<Visit> {
|
||||||
|
if let Some((_name, _builder)) = self.map.as_mut() {
|
||||||
|
for i in 0..m.len {
|
||||||
|
let bt = BlockTime {
|
||||||
|
block: m.data_begin + i,
|
||||||
|
time: m.time,
|
||||||
|
};
|
||||||
|
let (_, builder) = self.map.as_mut().unwrap();
|
||||||
|
builder.push_value(self.w, m.thin_begin + i, bt)?;
|
||||||
|
}
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
} else {
|
||||||
|
let msg = format!("Mapping tags must appear within a <def> or <device> tag.");
|
||||||
|
Err(anyhow!(msg))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ref_shared(&mut self, name: &str) -> Result<Visit> {
|
||||||
|
if self.current_dev.is_none() {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"<ref> tags may only occur within <device> sections."
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(leaves) = self.sub_trees.get(name) {
|
||||||
|
// We could be in a <def> or <device>
|
||||||
|
if let Some((_name, builder)) = self.map.as_mut() {
|
||||||
|
builder.push_nodes(self.w, leaves)?;
|
||||||
|
} else {
|
||||||
|
let msg = format!(
|
||||||
|
"<ref name=\"{}\"> tag must be within either a <def> or <device> section",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
return Err(anyhow!(msg));
|
||||||
|
}
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
} else {
|
||||||
|
let msg = format!("Couldn't find sub tree '{}'.", name);
|
||||||
|
Err(anyhow!(msg))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn eof(&mut self) -> Result<Visit> {
|
||||||
|
// FIXME: build the rest of the device trees
|
||||||
|
Ok(Visit::Continue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
/*
|
||||||
|
/// Writes a data space map to disk. Returns the space map root that needs
|
||||||
|
/// to be written to the superblock.
|
||||||
|
fn build_data_sm(batcher: WriteBatcher, sm: Box<dyn SpaceMap>) -> Result<Vec<u8>> {
|
||||||
|
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -13,10 +208,101 @@ pub struct ThinRestoreOptions<'a> {
|
|||||||
pub report: Arc<Report>,
|
pub report: Arc<Report>,
|
||||||
}
|
}
|
||||||
|
|
||||||
//------------------------------------------
|
struct Context {
|
||||||
|
report: Arc<Report>,
|
||||||
|
engine: Arc<dyn IoEngine + Send + Sync>,
|
||||||
|
}
|
||||||
|
|
||||||
pub fn restore(_opts: ThinRestoreOptions) -> Result<()> {
|
const MAX_CONCURRENT_IO: u32 = 1024;
|
||||||
todo!();
|
|
||||||
|
fn new_context(opts: &ThinRestoreOptions) -> Result<Context> {
|
||||||
|
let engine: Arc<dyn IoEngine + Send + Sync>;
|
||||||
|
|
||||||
|
if opts.async_io {
|
||||||
|
engine = Arc::new(AsyncIoEngine::new(opts.output, MAX_CONCURRENT_IO, true)?);
|
||||||
|
} else {
|
||||||
|
let nr_threads = std::cmp::max(8, num_cpus::get() * 2);
|
||||||
|
engine = Arc::new(SyncIoEngine::new(opts.output, nr_threads, true)?);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Context {
|
||||||
|
report: opts.report.clone(),
|
||||||
|
engine,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
//------------------------------------------
|
||||||
|
|
||||||
|
pub fn restore(opts: ThinRestoreOptions) -> Result<()> {
|
||||||
|
let input = OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.write(false)
|
||||||
|
.open(opts.input)?;
|
||||||
|
|
||||||
|
let ctx = new_context(&opts)?;
|
||||||
|
let max_count = u32::MAX;
|
||||||
|
|
||||||
|
let sm = core_sm(ctx.engine.get_nr_blocks(), max_count);
|
||||||
|
let mut w = WriteBatcher::new(ctx.engine.clone(), sm.clone(), ctx.engine.get_batch_size());
|
||||||
|
let mut pass = Pass1::new(&mut w);
|
||||||
|
xml::read(input, &mut pass)?;
|
||||||
|
let pass = pass.get_result();
|
||||||
|
|
||||||
|
// Build the device details tree.
|
||||||
|
let mut details_builder: Builder<DeviceDetail> = Builder::new(Box::new(NoopRC {}));
|
||||||
|
for (thin_id, (detail, _)) in &pass.devices {
|
||||||
|
details_builder.push_value(&mut w, *thin_id as u64, *detail)?;
|
||||||
|
}
|
||||||
|
let details_root = details_builder.complete(&mut w)?;
|
||||||
|
|
||||||
|
// Build the individual mapping trees that make up the bottom layer.
|
||||||
|
let mut devs: BTreeMap<u32, u64> = BTreeMap::new();
|
||||||
|
for (thin_id, (_, nodes)) in &pass.devices {
|
||||||
|
ctx.report
|
||||||
|
.info(&format!("building btree for device {}", thin_id));
|
||||||
|
let mut builder: Builder<BlockTime> = Builder::new(Box::new(NoopRC {}));
|
||||||
|
builder.push_leaves(&mut w, nodes)?;
|
||||||
|
let root = builder.complete(&mut w)?;
|
||||||
|
devs.insert(*thin_id, root);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the top level mapping tree
|
||||||
|
let mut builder: Builder<u64> = Builder::new(Box::new(NoopRC {}));
|
||||||
|
for (thin_id, root) in devs {
|
||||||
|
builder.push_value(&mut w, thin_id as u64, root)?;
|
||||||
|
}
|
||||||
|
let mapping_root = builder.complete(&mut w)?;
|
||||||
|
|
||||||
|
// Build data space map
|
||||||
|
|
||||||
|
// FIXME: I think we need to decrement the shared leaves
|
||||||
|
// Build metadata space map
|
||||||
|
|
||||||
|
w.flush()?;
|
||||||
|
|
||||||
|
// Write the superblock
|
||||||
|
if let Some(xml_sb) = pass.sb {
|
||||||
|
let sb = superblock::Superblock {
|
||||||
|
flags: SuperblockFlags { needs_check: false },
|
||||||
|
block: SUPERBLOCK_LOCATION,
|
||||||
|
version: 2,
|
||||||
|
time: xml_sb.time as u32,
|
||||||
|
transaction_id: xml_sb.transaction,
|
||||||
|
metadata_snap: 0,
|
||||||
|
data_sm_root: vec![0; SPACE_MAP_ROOT_SIZE],
|
||||||
|
metadata_sm_root: vec![0; SPACE_MAP_ROOT_SIZE],
|
||||||
|
mapping_root,
|
||||||
|
details_root,
|
||||||
|
data_block_size: xml_sb.data_block_size,
|
||||||
|
nr_metadata_blocks: ctx.engine.get_nr_blocks(),
|
||||||
|
};
|
||||||
|
|
||||||
|
write_superblock(ctx.engine.as_ref(), SUPERBLOCK_LOCATION, &sb)?;
|
||||||
|
} else {
|
||||||
|
return Err(anyhow!("No superblock found in xml file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
@ -1,10 +1,18 @@
|
|||||||
use crate::io_engine::*;
|
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
|
use byteorder::{LittleEndian, WriteBytesExt};
|
||||||
use nom::{bytes::complete::*, number::complete::*, IResult};
|
use nom::{bytes::complete::*, number::complete::*, IResult};
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
use crate::checksum::*;
|
||||||
|
use crate::io_engine::*;
|
||||||
|
|
||||||
|
//----------------------------------------
|
||||||
|
|
||||||
|
pub const MAGIC: u64 = 27022010;
|
||||||
pub const SUPERBLOCK_LOCATION: u64 = 0;
|
pub const SUPERBLOCK_LOCATION: u64 = 0;
|
||||||
//const UUID_SIZE: usize = 16;
|
const UUID_SIZE: usize = 16;
|
||||||
const SPACE_MAP_ROOT_SIZE: usize = 128;
|
pub const SPACE_MAP_ROOT_SIZE: usize = 128;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct SuperblockFlags {
|
pub struct SuperblockFlags {
|
||||||
@ -35,36 +43,9 @@ pub struct Superblock {
|
|||||||
pub mapping_root: u64,
|
pub mapping_root: u64,
|
||||||
pub details_root: u64,
|
pub details_root: u64,
|
||||||
pub data_block_size: u32,
|
pub data_block_size: u32,
|
||||||
|
pub nr_metadata_blocks: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
pub enum CheckSeverity {
|
|
||||||
Fatal,
|
|
||||||
NonFatal,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait CheckError {
|
|
||||||
fn severity(&self) -> CheckSeverity;
|
|
||||||
fn block(&self) -> u64;
|
|
||||||
fn sub_errors(&self) -> Vec<Box<dyn CheckError>>;
|
|
||||||
}
|
|
||||||
|
|
||||||
enum ErrorType {
|
|
||||||
BadChecksum,
|
|
||||||
BadBlockType(&'static str),
|
|
||||||
BadBlock(u64),
|
|
||||||
BadVersion(u32),
|
|
||||||
MetadataSnapOutOfBounds(u64),
|
|
||||||
MappingRootOutOfBounds(u64),
|
|
||||||
DetailsRootOutOfBounds(u64),
|
|
||||||
}
|
|
||||||
|
|
||||||
struct SuperblockError {
|
|
||||||
severity: CheckSeverity,
|
|
||||||
kind: ErrorType,
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
|
fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
|
||||||
let (i, _csum) = le_u32(data)?;
|
let (i, _csum) = le_u32(data)?;
|
||||||
let (i, flags) = le_u32(i)?;
|
let (i, flags) = le_u32(i)?;
|
||||||
@ -81,7 +62,7 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
|
|||||||
let (i, details_root) = le_u64(i)?;
|
let (i, details_root) = le_u64(i)?;
|
||||||
let (i, data_block_size) = le_u32(i)?;
|
let (i, data_block_size) = le_u32(i)?;
|
||||||
let (i, _metadata_block_size) = le_u32(i)?;
|
let (i, _metadata_block_size) = le_u32(i)?;
|
||||||
let (i, _metadata_nr_blocks) = le_u64(i)?;
|
let (i, nr_metadata_blocks) = le_u64(i)?;
|
||||||
|
|
||||||
Ok((
|
Ok((
|
||||||
i,
|
i,
|
||||||
@ -100,6 +81,7 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
|
|||||||
mapping_root,
|
mapping_root,
|
||||||
details_root,
|
details_root,
|
||||||
data_block_size,
|
data_block_size,
|
||||||
|
nr_metadata_blocks,
|
||||||
},
|
},
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
@ -115,3 +97,51 @@ pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result<Superblock> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
//------------------------------
|
//------------------------------
|
||||||
|
|
||||||
|
fn pack_superblock<W: WriteBytesExt>(sb: &Superblock, w: &mut W) -> Result<()> {
|
||||||
|
// checksum, which we don't know yet
|
||||||
|
w.write_u32::<LittleEndian>(0)?;
|
||||||
|
|
||||||
|
// flags
|
||||||
|
if sb.flags.needs_check {
|
||||||
|
w.write_u32::<LittleEndian>(0x1)?;
|
||||||
|
} else {
|
||||||
|
w.write_u32::<LittleEndian>(0)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
w.write_u64::<LittleEndian>(sb.block)?;
|
||||||
|
w.write_all(&vec![0; UUID_SIZE])?;
|
||||||
|
w.write_u64::<LittleEndian>(MAGIC)?;
|
||||||
|
w.write_u32::<LittleEndian>(sb.version)?;
|
||||||
|
w.write_u32::<LittleEndian>(sb.time)?;
|
||||||
|
w.write_u64::<LittleEndian>(sb.transaction_id)?;
|
||||||
|
w.write_u64::<LittleEndian>(sb.metadata_snap)?;
|
||||||
|
w.write_all(&vec![0; SPACE_MAP_ROOT_SIZE])?; // data sm root
|
||||||
|
w.write_all(&vec![0; SPACE_MAP_ROOT_SIZE])?; // metadata sm root
|
||||||
|
w.write_u64::<LittleEndian>(sb.mapping_root)?;
|
||||||
|
w.write_u64::<LittleEndian>(sb.details_root)?;
|
||||||
|
w.write_u32::<LittleEndian>(sb.data_block_size)?;
|
||||||
|
w.write_u32::<LittleEndian>(BLOCK_SIZE as u32)?;
|
||||||
|
w.write_u64::<LittleEndian>(sb.nr_metadata_blocks)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn write_superblock(engine: &dyn IoEngine, _loc: u64, sb: &Superblock) -> Result<()> {
|
||||||
|
let b = Block::zeroed(SUPERBLOCK_LOCATION);
|
||||||
|
|
||||||
|
// pack the superblock
|
||||||
|
{
|
||||||
|
let mut cursor = Cursor::new(b.get_data());
|
||||||
|
pack_superblock(sb, &mut cursor)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// calculate the checksum
|
||||||
|
write_checksum(b.get_data(), BT::SUPERBLOCK)?;
|
||||||
|
|
||||||
|
// write
|
||||||
|
engine.write(&b)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
//------------------------------
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
use anyhow::Result;
|
use anyhow::{anyhow, Result};
|
||||||
use std::{borrow::Cow, fmt::Display, io::prelude::*, io::BufReader, io::Write};
|
use std::{borrow::Cow, fmt::Display, io::prelude::*, io::BufReader, io::Write};
|
||||||
|
|
||||||
use quick_xml::events::attributes::Attribute;
|
use quick_xml::events::attributes::Attribute;
|
||||||
@ -46,9 +46,11 @@ pub trait MetadataVisitor {
|
|||||||
fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit>;
|
fn superblock_b(&mut self, sb: &Superblock) -> Result<Visit>;
|
||||||
fn superblock_e(&mut self) -> Result<Visit>;
|
fn superblock_e(&mut self) -> Result<Visit>;
|
||||||
|
|
||||||
|
// Defines a shared sub tree. May only contain a 'map' (no 'ref' allowed).
|
||||||
fn def_shared_b(&mut self, name: &str) -> Result<Visit>;
|
fn def_shared_b(&mut self, name: &str) -> Result<Visit>;
|
||||||
fn def_shared_e(&mut self) -> Result<Visit>;
|
fn def_shared_e(&mut self) -> Result<Visit>;
|
||||||
|
|
||||||
|
// A device contains a number of 'map' or 'ref' items.
|
||||||
fn device_b(&mut self, d: &Device) -> Result<Visit>;
|
fn device_b(&mut self, d: &Device) -> Result<Visit>;
|
||||||
fn device_e(&mut self) -> Result<Visit>;
|
fn device_e(&mut self) -> Result<Visit>;
|
||||||
|
|
||||||
@ -207,8 +209,9 @@ fn bad_attr<T>(_tag: &str, _attr: &[u8]) -> Result<T> {
|
|||||||
todo!();
|
todo!();
|
||||||
}
|
}
|
||||||
|
|
||||||
fn missing_attr<T>(_tag: &str, _attr: &str) -> Result<T> {
|
fn missing_attr<T>(tag: &str, attr: &str) -> Result<T> {
|
||||||
todo!();
|
let msg = format!("missing attribute '{}' for tag '{}", attr, tag);
|
||||||
|
Err(anyhow!(msg))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_attr<T>(tag: &str, name: &str, maybe_v: Option<T>) -> Result<T> {
|
fn check_attr<T>(tag: &str, name: &str, maybe_v: Option<T>) -> Result<T> {
|
||||||
@ -257,6 +260,22 @@ fn parse_superblock(e: &BytesStart) -> Result<Superblock> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_def(e: &BytesStart, tag: &str) -> Result<String> {
|
||||||
|
let mut name: Option<String> = None;
|
||||||
|
|
||||||
|
for a in e.attributes() {
|
||||||
|
let kv = a.unwrap();
|
||||||
|
match kv.key {
|
||||||
|
b"name" => {
|
||||||
|
name = Some(string_val(&kv));
|
||||||
|
}
|
||||||
|
_ => return bad_attr(tag, kv.key),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(name.unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_device(e: &BytesStart) -> Result<Device> {
|
fn parse_device(e: &BytesStart) -> Result<Device> {
|
||||||
let mut dev_id: Option<u32> = None;
|
let mut dev_id: Option<u32> = None;
|
||||||
let mut mapped_blocks: Option<u64> = None;
|
let mut mapped_blocks: Option<u64> = None;
|
||||||
@ -348,16 +367,19 @@ where
|
|||||||
Ok(Event::Start(ref e)) => match e.name() {
|
Ok(Event::Start(ref e)) => match e.name() {
|
||||||
b"superblock" => visitor.superblock_b(&parse_superblock(e)?),
|
b"superblock" => visitor.superblock_b(&parse_superblock(e)?),
|
||||||
b"device" => visitor.device_b(&parse_device(e)?),
|
b"device" => visitor.device_b(&parse_device(e)?),
|
||||||
|
b"def" => visitor.def_shared_b(&parse_def(e, "def")?),
|
||||||
_ => todo!(),
|
_ => todo!(),
|
||||||
},
|
},
|
||||||
Ok(Event::End(ref e)) => match e.name() {
|
Ok(Event::End(ref e)) => match e.name() {
|
||||||
b"superblock" => visitor.superblock_e(),
|
b"superblock" => visitor.superblock_e(),
|
||||||
b"device" => visitor.device_e(),
|
b"device" => visitor.device_e(),
|
||||||
|
b"def" => visitor.def_shared_e(),
|
||||||
_ => todo!(),
|
_ => todo!(),
|
||||||
},
|
},
|
||||||
Ok(Event::Empty(ref e)) => match e.name() {
|
Ok(Event::Empty(ref e)) => match e.name() {
|
||||||
b"single_mapping" => visitor.map(&parse_single_map(e)?),
|
b"single_mapping" => visitor.map(&parse_single_map(e)?),
|
||||||
b"range_mapping" => visitor.map(&parse_range_map(e)?),
|
b"range_mapping" => visitor.map(&parse_range_map(e)?),
|
||||||
|
b"ref" => visitor.ref_shared(&parse_def(e, "ref")?),
|
||||||
_ => todo!(),
|
_ => todo!(),
|
||||||
},
|
},
|
||||||
Ok(Event::Text(_)) => Ok(Visit::Continue),
|
Ok(Event::Text(_)) => Ok(Visit::Continue),
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
|
use std::collections::BTreeSet;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use crate::checksum;
|
use crate::checksum;
|
||||||
@ -7,12 +8,16 @@ use crate::pdata::space_map::*;
|
|||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct WriteBatcher {
|
pub struct WriteBatcher {
|
||||||
engine: Arc<dyn IoEngine + Send + Sync>,
|
pub engine: Arc<dyn IoEngine + Send + Sync>,
|
||||||
sm: Arc<Mutex<dyn SpaceMap>>,
|
|
||||||
|
// FIXME: this doesn't need to be in a mutex
|
||||||
|
pub sm: Arc<Mutex<dyn SpaceMap>>,
|
||||||
|
|
||||||
batch_size: usize,
|
batch_size: usize,
|
||||||
queue: Vec<Block>,
|
queue: Vec<Block>,
|
||||||
|
allocations: BTreeSet<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WriteBatcher {
|
impl WriteBatcher {
|
||||||
@ -26,10 +31,11 @@ impl WriteBatcher {
|
|||||||
sm,
|
sm,
|
||||||
batch_size,
|
batch_size,
|
||||||
queue: Vec::with_capacity(batch_size),
|
queue: Vec::with_capacity(batch_size),
|
||||||
|
allocations: BTreeSet::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn alloc(&mut self) -> Result<u64> {
|
pub fn alloc(&mut self) -> Result<Block> {
|
||||||
let mut sm = self.sm.lock().unwrap();
|
let mut sm = self.sm.lock().unwrap();
|
||||||
let b = sm.alloc()?;
|
let b = sm.alloc()?;
|
||||||
|
|
||||||
@ -37,23 +43,51 @@ impl WriteBatcher {
|
|||||||
return Err(anyhow!("out of metadata space"));
|
return Err(anyhow!("out of metadata space"));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(b.unwrap())
|
Ok(Block::new(b.unwrap()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn clear_allocations(&mut self) -> BTreeSet<u64> {
|
||||||
|
let mut tmp = BTreeSet::new();
|
||||||
|
std::mem::swap(&mut tmp, &mut self.allocations);
|
||||||
|
tmp
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn write(&mut self, b: Block, kind: checksum::BT) -> Result<()> {
|
pub fn write(&mut self, b: Block, kind: checksum::BT) -> Result<()> {
|
||||||
checksum::write_checksum(&mut b.get_data(), kind)?;
|
checksum::write_checksum(&mut b.get_data(), kind)?;
|
||||||
|
|
||||||
if self.queue.len() == self.batch_size {
|
if self.queue.len() == self.batch_size {
|
||||||
self.flush()?;
|
let mut tmp = Vec::new();
|
||||||
|
std::mem::swap(&mut tmp, &mut self.queue);
|
||||||
|
self.flush_(tmp)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
self.queue.push(b);
|
self.queue.push(b);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn read(&mut self, blocknr: u64) -> Result<Block> {
|
||||||
|
for b in self.queue.iter().rev() {
|
||||||
|
if b.loc == blocknr {
|
||||||
|
let r = Block::new(b.loc);
|
||||||
|
r.get_data().copy_from_slice(b.get_data());
|
||||||
|
return Ok(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.engine
|
||||||
|
.read(blocknr)
|
||||||
|
.map_err(|_| anyhow!("read block error"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn flush_(&mut self, queue: Vec<Block>) -> Result<()> {
|
||||||
|
self.engine.write_many(&queue)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn flush(&mut self) -> Result<()> {
|
pub fn flush(&mut self) -> Result<()> {
|
||||||
self.engine.write_many(&self.queue)?;
|
let mut tmp = Vec::new();
|
||||||
self.queue.clear();
|
std::mem::swap(&mut tmp, &mut self.queue);
|
||||||
|
self.flush_(tmp)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use thinp::version::TOOLS_VERSION;
|
|
||||||
use duct::cmd;
|
use duct::cmd;
|
||||||
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
|
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
use anyhow::{Result};
|
use anyhow::Result;
|
||||||
use rand::prelude::*;
|
use rand::prelude::*;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
|
@ -4,14 +4,14 @@ use anyhow::Result;
|
|||||||
use duct::{cmd, Expression};
|
use duct::{cmd, Expression};
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::path::{PathBuf};
|
use std::path::PathBuf;
|
||||||
use std::str::from_utf8;
|
use std::str::from_utf8;
|
||||||
use thinp::file_utils;
|
use thinp::file_utils;
|
||||||
use thinp::io_engine::*;
|
use thinp::io_engine::*;
|
||||||
|
|
||||||
pub mod thin_xml_generator;
|
|
||||||
pub mod cache_xml_generator;
|
pub mod cache_xml_generator;
|
||||||
pub mod test_dir;
|
pub mod test_dir;
|
||||||
|
pub mod thin_xml_generator;
|
||||||
|
|
||||||
use crate::common::thin_xml_generator::{write_xml, SingleThinS};
|
use crate::common::thin_xml_generator::{write_xml, SingleThinS};
|
||||||
use test_dir::TestDir;
|
use test_dir::TestDir;
|
||||||
@ -273,7 +273,12 @@ pub fn set_needs_check(md: &PathBuf) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn generate_metadata_leaks(md: &PathBuf, nr_blocks: u64, expected: u32, actual: u32) -> Result<()> {
|
pub fn generate_metadata_leaks(
|
||||||
|
md: &PathBuf,
|
||||||
|
nr_blocks: u64,
|
||||||
|
expected: u32,
|
||||||
|
actual: u32,
|
||||||
|
) -> Result<()> {
|
||||||
let output = thin_generate_damage!(
|
let output = thin_generate_damage!(
|
||||||
"-o",
|
"-o",
|
||||||
&md,
|
&md,
|
||||||
@ -318,4 +323,3 @@ where
|
|||||||
assert_eq!(csum, md5(p)?);
|
assert_eq!(csum, md5(p)?);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2,8 +2,8 @@ use anyhow::Result;
|
|||||||
use thinp::version::TOOLS_VERSION;
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -68,4 +68,3 @@ fn dev_unspecified() -> Result<()> {
|
|||||||
assert!(stderr.contains("No input device provided"));
|
assert!(stderr.contains("No input device provided"));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use thinp::file_utils;
|
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
use std::io::{Write};
|
use std::io::Write;
|
||||||
use std::str::from_utf8;
|
use std::str::from_utf8;
|
||||||
|
use thinp::file_utils;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -27,7 +27,11 @@ fn dump_restore_cycle() -> Result<()> {
|
|||||||
let output = thin_dump!(&md).run()?;
|
let output = thin_dump!(&md).run()?;
|
||||||
|
|
||||||
let xml = td.mk_path("meta.xml");
|
let xml = td.mk_path("meta.xml");
|
||||||
let mut file = OpenOptions::new().read(false).write(true).create(true).open(&xml)?;
|
let mut file = OpenOptions::new()
|
||||||
|
.read(false)
|
||||||
|
.write(true)
|
||||||
|
.create(true)
|
||||||
|
.open(&xml)?;
|
||||||
file.write_all(&output.stdout[0..])?;
|
file.write_all(&output.stdout[0..])?;
|
||||||
drop(file);
|
drop(file);
|
||||||
|
|
||||||
@ -80,10 +84,23 @@ fn override_nr_data_blocks() -> Result<()> {
|
|||||||
fn repair_superblock() -> Result<()> {
|
fn repair_superblock() -> Result<()> {
|
||||||
let mut td = TestDir::new()?;
|
let mut td = TestDir::new()?;
|
||||||
let md = mk_valid_md(&mut td)?;
|
let md = mk_valid_md(&mut td)?;
|
||||||
let before = thin_dump!("--transaction-id=5", "--data-block-size=128", "--nr-data-blocks=4096000", &md).run()?;
|
let before = thin_dump!(
|
||||||
|
"--transaction-id=5",
|
||||||
|
"--data-block-size=128",
|
||||||
|
"--nr-data-blocks=4096000",
|
||||||
|
&md
|
||||||
|
)
|
||||||
|
.run()?;
|
||||||
damage_superblock(&md)?;
|
damage_superblock(&md)?;
|
||||||
|
|
||||||
let after = thin_dump!("--repair", "--transaction-id=5", "--data-block-size=128", "--nr-data-blocks=4096000", &md).run()?;
|
let after = thin_dump!(
|
||||||
|
"--repair",
|
||||||
|
"--transaction-id=5",
|
||||||
|
"--data-block-size=128",
|
||||||
|
"--nr-data-blocks=4096000",
|
||||||
|
&md
|
||||||
|
)
|
||||||
|
.run()?;
|
||||||
assert_eq!(after.stderr.len(), 0);
|
assert_eq!(after.stderr.len(), 0);
|
||||||
assert_eq!(before.stdout, after.stdout);
|
assert_eq!(before.stdout, after.stdout);
|
||||||
|
|
||||||
@ -95,7 +112,12 @@ fn missing_transaction_id() -> Result<()> {
|
|||||||
let mut td = TestDir::new()?;
|
let mut td = TestDir::new()?;
|
||||||
let md = mk_valid_md(&mut td)?;
|
let md = mk_valid_md(&mut td)?;
|
||||||
damage_superblock(&md)?;
|
damage_superblock(&md)?;
|
||||||
let stderr = run_fail(thin_dump!("--repair", "--data-block-size=128", "--nr-data-blocks=4096000", &md))?;
|
let stderr = run_fail(thin_dump!(
|
||||||
|
"--repair",
|
||||||
|
"--data-block-size=128",
|
||||||
|
"--nr-data-blocks=4096000",
|
||||||
|
&md
|
||||||
|
))?;
|
||||||
assert!(stderr.contains("transaction id"));
|
assert!(stderr.contains("transaction id"));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -105,7 +127,12 @@ fn missing_data_block_size() -> Result<()> {
|
|||||||
let mut td = TestDir::new()?;
|
let mut td = TestDir::new()?;
|
||||||
let md = mk_valid_md(&mut td)?;
|
let md = mk_valid_md(&mut td)?;
|
||||||
damage_superblock(&md)?;
|
damage_superblock(&md)?;
|
||||||
let stderr = run_fail(thin_dump!("--repair", "--transaction-id=5", "--nr-data-blocks=4096000", &md))?;
|
let stderr = run_fail(thin_dump!(
|
||||||
|
"--repair",
|
||||||
|
"--transaction-id=5",
|
||||||
|
"--nr-data-blocks=4096000",
|
||||||
|
&md
|
||||||
|
))?;
|
||||||
assert!(stderr.contains("data block size"));
|
assert!(stderr.contains("data block size"));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -115,7 +142,12 @@ fn missing_nr_data_blocks() -> Result<()> {
|
|||||||
let mut td = TestDir::new()?;
|
let mut td = TestDir::new()?;
|
||||||
let md = mk_valid_md(&mut td)?;
|
let md = mk_valid_md(&mut td)?;
|
||||||
damage_superblock(&md)?;
|
damage_superblock(&md)?;
|
||||||
let stderr = run_fail(thin_dump!("--repair", "--transaction-id=5", "--data-block-size=128", &md))?;
|
let stderr = run_fail(thin_dump!(
|
||||||
|
"--repair",
|
||||||
|
"--transaction-id=5",
|
||||||
|
"--data-block-size=128",
|
||||||
|
&md
|
||||||
|
))?;
|
||||||
assert!(stderr.contains("nr data blocks"));
|
assert!(stderr.contains("nr data blocks"));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -2,8 +2,8 @@ use anyhow::Result;
|
|||||||
use thinp::version::TOOLS_VERSION;
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
@ -2,8 +2,8 @@ use anyhow::Result;
|
|||||||
use thinp::version::TOOLS_VERSION;
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
@ -3,8 +3,8 @@ use std::str::from_utf8;
|
|||||||
use thinp::version::TOOLS_VERSION;
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -132,8 +132,7 @@ fn superblock_succeeds() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn missing_thing(flag1: &str, flag2: &str, pattern: &str) -> Result<()>
|
fn missing_thing(flag1: &str, flag2: &str, pattern: &str) -> Result<()> {
|
||||||
{
|
|
||||||
let mut td = TestDir::new()?;
|
let mut td = TestDir::new()?;
|
||||||
let md1 = mk_valid_md(&mut td)?;
|
let md1 = mk_valid_md(&mut td)?;
|
||||||
damage_superblock(&md1)?;
|
damage_superblock(&md1)?;
|
||||||
@ -145,15 +144,27 @@ fn missing_thing(flag1: &str, flag2: &str, pattern: &str) -> Result<()>
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn missing_transaction_id() -> Result<()> {
|
fn missing_transaction_id() -> Result<()> {
|
||||||
missing_thing("--data-block-size=128", "--nr-data-blocks=4096000", "transaction id")
|
missing_thing(
|
||||||
|
"--data-block-size=128",
|
||||||
|
"--nr-data-blocks=4096000",
|
||||||
|
"transaction id",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn missing_data_block_size() -> Result<()> {
|
fn missing_data_block_size() -> Result<()> {
|
||||||
missing_thing("--transaction-id=5", "--nr-data-blocks=4096000", "data block size")
|
missing_thing(
|
||||||
|
"--transaction-id=5",
|
||||||
|
"--nr-data-blocks=4096000",
|
||||||
|
"data block size",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn missing_nr_data_blocks() -> Result<()> {
|
fn missing_nr_data_blocks() -> Result<()> {
|
||||||
missing_thing("--transaction-id=5", "--data-block-size=128", "nr data blocks")
|
missing_thing(
|
||||||
|
"--transaction-id=5",
|
||||||
|
"--data-block-size=128",
|
||||||
|
"nr data blocks",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
@ -4,8 +4,8 @@ use thinp::file_utils;
|
|||||||
use thinp::version::TOOLS_VERSION;
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
|
@ -2,8 +2,8 @@ use anyhow::Result;
|
|||||||
use thinp::version::TOOLS_VERSION;
|
use thinp::version::TOOLS_VERSION;
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
//------------------------------------------
|
//------------------------------------------
|
||||||
|
|
||||||
@ -54,7 +54,16 @@ fn valid_region_format_should_pass() -> Result<()> {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn invalid_regions_should_fail() -> Result<()> {
|
fn invalid_regions_should_fail() -> Result<()> {
|
||||||
let invalid_regions = ["23,7890", "23..six", "found..7890", "89..88", "89..89", "89..", "", "89...99"];
|
let invalid_regions = [
|
||||||
|
"23,7890",
|
||||||
|
"23..six",
|
||||||
|
"found..7890",
|
||||||
|
"89..88",
|
||||||
|
"89..89",
|
||||||
|
"89..",
|
||||||
|
"",
|
||||||
|
"89...99",
|
||||||
|
];
|
||||||
for r in &invalid_regions {
|
for r in &invalid_regions {
|
||||||
let mut td = TestDir::new()?;
|
let mut td = TestDir::new()?;
|
||||||
let md = mk_valid_md(&mut td)?;
|
let md = mk_valid_md(&mut td)?;
|
||||||
|
@ -3,16 +3,14 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
|
|||||||
use rand::prelude::*;
|
use rand::prelude::*;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
|
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
|
||||||
use std::path::{Path};
|
use std::path::Path;
|
||||||
|
|
||||||
use thinp::file_utils;
|
use thinp::file_utils;
|
||||||
use thinp::thin::xml::{self, Visit};
|
use thinp::thin::xml::{self, Visit};
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
use common::test_dir::*;
|
use common::test_dir::*;
|
||||||
use common::thin_xml_generator::{
|
use common::thin_xml_generator::{write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen};
|
||||||
write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen
|
|
||||||
};
|
|
||||||
|
|
||||||
//------------------------------------
|
//------------------------------------
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user