emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

work in progress

Browse Source
This commit is contained in:
Joe Thornber
2020-11-04 12:38:35 +00:00
parent 37ea0280df
commit 1ae62adec6
15 changed files with 1722 additions and 791 deletions
Download Patch File Download Diff File Expand all files Collapse all files

584
src/pdata/btree.rs
View File

@@ -2,24 +2,28 @@ use anyhow::anyhow;
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use data_encoding::BASE64;
use nom::{number::complete::*, IResult};
use std::collections::BTreeMap;
use std::fmt;
use std::sync::{Arc, Mutex};
use thiserror::Error;
use threadpool::ThreadPool;
use crate::checksum;
use crate::io_engine::*;
use crate::pack::vm;
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
//------------------------------------------
#[derive(Clone, Debug, PartialEq)]
pub struct KeyRange {
start: Option<u64>,
end: Option<u64>, // This is the one-past-the-end value
pub start: Option<u64>,
pub end: Option<u64>, // This is the one-past-the-end value
}
impl KeyRange {
pub fn new() -> KeyRange {
KeyRange {
start: None,
end: None,
}
}
}
impl fmt::Display for KeyRange {
@@ -190,7 +194,7 @@ fn split_one(path: &Vec<u64>, kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRan
}
}
fn split_key_ranges(path: &Vec<u64>, kr: &KeyRange, keys: &[u64]) -> Result<Vec<KeyRange>> {
pub fn split_key_ranges(path: &Vec<u64>, kr: &KeyRange, keys: &[u64]) -> Result<Vec<KeyRange>> {
let mut krs = Vec::with_capacity(keys.len());
if keys.len() == 0 {
@@ -352,7 +356,7 @@ pub fn node_err(path: &Vec<u64>, msg: &str) -> BTreeError {
)
}
fn node_err_s(path: &Vec<u64>, msg: String) -> BTreeError {
pub fn node_err_s(path: &Vec<u64>, msg: String) -> BTreeError {
BTreeError::Path(path.clone(), Box::new(BTreeError::NodeError(msg)))
}
@@ -579,565 +583,3 @@ pub fn unpack_node<V: Unpack>(
}
//------------------------------------------
pub trait NodeVisitor<V: Unpack> {
// &self is deliberately non mut to allow the walker to use multiple threads.
fn visit(
&self,
path: &Vec<u64>,
keys: &KeyRange,
header: &NodeHeader,
keys: &[u64],
values: &[V],
) -> Result<()>;
// Nodes may be shared and thus visited multiple times. The walker avoids
// doing repeated IO, but it does call this method to keep the visitor up to
// date.
fn visit_again(&self, path: &Vec<u64>, b: u64) -> Result<()>;
fn end_walk(&self) -> Result<()>;
}
#[derive(Clone)]
pub struct BTreeWalker {
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
fails: Arc<Mutex<BTreeMap<u64, BTreeError>>>,
ignore_non_fatal: bool,
}
impl BTreeWalker {
pub fn new(engine: Arc<dyn IoEngine + Send + Sync>, ignore_non_fatal: bool) -> BTreeWalker {
let nr_blocks = engine.get_nr_blocks() as usize;
let r: BTreeWalker = BTreeWalker {
engine,
sm: Arc::new(Mutex::new(RestrictedSpaceMap::new(nr_blocks as u64))),
fails: Arc::new(Mutex::new(BTreeMap::new())),
ignore_non_fatal,
};
r
}
pub fn new_with_sm(
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
ignore_non_fatal: bool,
) -> Result<BTreeWalker> {
{
let sm = sm.lock().unwrap();
assert_eq!(sm.get_nr_blocks().unwrap(), engine.get_nr_blocks());
}
Ok(BTreeWalker {
engine,
sm,
fails: Arc::new(Mutex::new(BTreeMap::new())),
ignore_non_fatal,
})
}
fn failed(&self, b: u64) -> Option<BTreeError> {
let fails = self.fails.lock().unwrap();
match fails.get(&b) {
None => None,
Some(e) => Some(e.clone()),
}
}
fn set_fail(&self, b: u64, err: BTreeError) {
// FIXME: should we monitor the size of fails, and abort if too many errors?
let mut fails = self.fails.lock().unwrap();
fails.insert(b, err);
}
// Atomically increments the ref count, and returns the _old_ count.
fn sm_inc(&self, b: u64) -> u32 {
let mut sm = self.sm.lock().unwrap();
let count = sm.get(b).unwrap();
sm.inc(b, 1).unwrap();
count
}
fn build_aggregate(&self, b: u64, errs: Vec<BTreeError>) -> Result<()> {
match errs.len() {
0 => Ok(()),
1 => {
let e = errs[0].clone();
self.set_fail(b, e.clone());
Err(e)
}
_ => {
let e = aggregate_error(errs);
self.set_fail(b, e.clone());
Err(e)
}
}
}
fn walk_nodes<NV, V>(
&self,
path: &mut Vec<u64>,
visitor: &NV,
krs: &[KeyRange],
bs: &[u64],
) -> Vec<BTreeError>
where
NV: NodeVisitor<V>,
V: Unpack,
{
assert_eq!(krs.len(), bs.len());
let mut errs: Vec<BTreeError> = Vec::new();
let mut blocks = Vec::with_capacity(bs.len());
let mut filtered_krs = Vec::with_capacity(krs.len());
for i in 0..bs.len() {
if self.sm_inc(bs[i]) == 0 {
// Node not yet seen
blocks.push(bs[i]);
filtered_krs.push(krs[i].clone());
} else {
// This node has already been checked ...
match self.failed(bs[i]) {
None => {
// ... it was clean.
if let Err(e) = visitor.visit_again(path, bs[i]) {
// ... but the visitor isn't happy
errs.push(e.clone());
}
}
Some(e) => {
// ... there was an error
errs.push(e.clone());
}
}
}
}
match self.engine.read_many(&blocks[0..]) {
Err(_) => {
// IO completely failed, error every block
for (i, b) in blocks.iter().enumerate() {
let e = io_err(path).keys_context(&filtered_krs[i]);
errs.push(e.clone());
self.set_fail(*b, e);
}
}
Ok(rblocks) => {
let mut i = 0;
for rb in rblocks {
match rb {
Err(_) => {
let e = io_err(path).keys_context(&filtered_krs[i]);
errs.push(e.clone());
self.set_fail(blocks[i], e);
}
Ok(b) => match self.walk_node(path, visitor, &filtered_krs[i], &b, false) {
Err(e) => {
errs.push(e);
}
Ok(()) => {}
},
}
i += 1;
}
}
}
errs
}
fn walk_node_<NV, V>(
&self,
path: &mut Vec<u64>,
visitor: &NV,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
use Node::*;
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(node_err_s(
path,
format!("checksum failed for node {}, {:?}", b.loc, bt),
)
.keys_context(kr));
}
let node = unpack_node::<V>(path, &b.get_data(), self.ignore_non_fatal, is_root)?;
match node {
Internal { keys, values, .. } => {
let krs = split_key_ranges(path, &kr, &keys)?;
let errs = self.walk_nodes(path, visitor, &krs, &values);
return self.build_aggregate(b.loc, errs);
}
Leaf {
header,
keys,
values,
} => {
if let Err(e) = visitor.visit(path, &kr, &header, &keys, &values) {
let e = BTreeError::Path(path.clone(), Box::new(e.clone()));
self.set_fail(b.loc, e.clone());
return Err(e);
}
}
}
Ok(())
}
fn walk_node<NV, V>(
&self,
path: &mut Vec<u64>,
visitor: &NV,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
path.push(b.loc);
let r = self.walk_node_(path, visitor, kr, b, is_root);
path.pop();
visitor.end_walk()?;
r
}
pub fn walk<NV, V>(&self, path: &mut Vec<u64>, visitor: &NV, root: u64) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
if self.sm_inc(root) > 0 {
if let Some(e) = self.failed(root) {
Err(e.clone())
} else {
visitor.visit_again(path, root)
}
} else {
let root = self.engine.read(root).map_err(|_| io_err(path))?;
let kr = KeyRange {
start: None,
end: None,
};
self.walk_node(path, visitor, &kr, &root, true)
}
}
}
//--------------------------------
fn walk_node_threaded_<NV, V>(
w: Arc<BTreeWalker>,
path: &mut Vec<u64>,
pool: &ThreadPool,
visitor: Arc<NV>,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
use Node::*;
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(node_err_s(
path,
format!("checksum failed for node {}, {:?}", b.loc, bt),
)
.keys_context(kr));
}
let node = unpack_node::<V>(path, &b.get_data(), w.ignore_non_fatal, is_root)?;
match node {
Internal { keys, values, .. } => {
let krs = split_key_ranges(path, &kr, &keys)?;
let errs = walk_nodes_threaded(w.clone(), path, pool, visitor, &krs, &values);
return w.build_aggregate(b.loc, errs);
}
Leaf {
header,
keys,
values,
} => {
visitor.visit(path, kr, &header, &keys, &values)?;
}
}
Ok(())
}
fn walk_node_threaded<NV, V>(
w: Arc<BTreeWalker>,
path: &mut Vec<u64>,
pool: &ThreadPool,
visitor: Arc<NV>,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
path.push(b.loc);
let r = walk_node_threaded_(w, path, pool, visitor.clone(), kr, b, is_root);
path.pop();
visitor.end_walk()?;
r
}
fn walk_nodes_threaded<NV, V>(
w: Arc<BTreeWalker>,
path: &mut Vec<u64>,
pool: &ThreadPool,
visitor: Arc<NV>,
krs: &[KeyRange],
bs: &[u64],
) -> Vec<BTreeError>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
assert_eq!(krs.len(), bs.len());
let mut errs: Vec<BTreeError> = Vec::new();
let mut blocks = Vec::with_capacity(bs.len());
let mut filtered_krs = Vec::with_capacity(krs.len());
for i in 0..bs.len() {
if w.sm_inc(bs[i]) == 0 {
// Node not yet seen
blocks.push(bs[i]);
filtered_krs.push(krs[i].clone());
} else {
// This node has already been checked ...
match w.failed(bs[i]) {
None => {
// ... it was clean.
if let Err(e) = visitor.visit_again(path, bs[i]) {
// ... but the visitor isn't happy
errs.push(e.clone());
}
}
Some(e) => {
// ... there was an error
errs.push(e.clone());
}
}
}
}
match w.engine.read_many(&blocks[0..]) {
Err(_) => {
// IO completely failed error every block
for (i, b) in blocks.iter().enumerate() {
let e = io_err(path).keys_context(&filtered_krs[i]);
errs.push(e.clone());
w.set_fail(*b, e);
}
}
Ok(rblocks) => {
let mut i = 0;
let errs = Arc::new(Mutex::new(Vec::new()));
for rb in rblocks {
match rb {
Err(_) => {
let e = io_err(path).keys_context(&filtered_krs[i]);
let mut errs = errs.lock().unwrap();
errs.push(e.clone());
w.set_fail(blocks[i], e);
}
Ok(b) => {
let w = w.clone();
let visitor = visitor.clone();
let kr = filtered_krs[i].clone();
let errs = errs.clone();
let mut path = path.clone();
pool.execute(move || {
match w.walk_node(&mut path, visitor.as_ref(), &kr, &b, false) {
Err(e) => {
let mut errs = errs.lock().unwrap();
errs.push(e);
}
Ok(()) => {}
}
});
}
}
i += 1;
}
pool.join();
}
}
errs
}
pub fn walk_threaded<NV, V>(
path: &mut Vec<u64>,
w: Arc<BTreeWalker>,
pool: &ThreadPool,
visitor: Arc<NV>,
root: u64,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
if w.sm_inc(root) > 0 {
if let Some(e) = w.failed(root) {
Err(e.clone())
} else {
visitor.visit_again(path, root)
}
} else {
let root = w.engine.read(root).map_err(|_| io_err(path))?;
let kr = KeyRange {
start: None,
end: None,
};
walk_node_threaded(w, path, pool, visitor, &kr, &root, true)
}
}
//------------------------------------------
struct ValueCollector<V> {
values: Mutex<BTreeMap<u64, V>>,
}
impl<V> ValueCollector<V> {
fn new() -> ValueCollector<V> {
ValueCollector {
values: Mutex::new(BTreeMap::new()),
}
}
}
// FIXME: should we be using Copy rather than clone? (Yes)
impl<V: Unpack + Copy> NodeVisitor<V> for ValueCollector<V> {
fn visit(
&self,
_path: &Vec<u64>,
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
values: &[V],
) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], values[n].clone());
}
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> Result<()> {
Ok(())
}
fn end_walk(&self) -> Result<()> {
Ok(())
}
}
pub fn btree_to_map<V: Unpack + Copy>(
path: &mut Vec<u64>,
engine: Arc<dyn IoEngine + Send + Sync>,
ignore_non_fatal: bool,
root: u64,
) -> Result<BTreeMap<u64, V>> {
let walker = BTreeWalker::new(engine, ignore_non_fatal);
let visitor = ValueCollector::<V>::new();
walker.walk(path, &visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}
pub fn btree_to_map_with_sm<V: Unpack + Copy>(
path: &mut Vec<u64>,
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
ignore_non_fatal: bool,
root: u64,
) -> Result<BTreeMap<u64, V>> {
let walker = BTreeWalker::new_with_sm(engine, sm, ignore_non_fatal)?;
let visitor = ValueCollector::<V>::new();
walker.walk(path, &visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}
//------------------------------------------
struct ValuePathCollector<V> {
values: Mutex<BTreeMap<u64, (Vec<u64>, V)>>,
}
impl<V> ValuePathCollector<V> {
fn new() -> ValuePathCollector<V> {
ValuePathCollector {
values: Mutex::new(BTreeMap::new()),
}
}
}
impl<V: Unpack + Clone> NodeVisitor<V> for ValuePathCollector<V> {
fn visit(
&self,
path: &Vec<u64>,
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
values: &[V],
) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], (path.clone(), values[n].clone()));
}
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> Result<()> {
Ok(())
}
fn end_walk(&self) -> Result<()> {
Ok(())
}
}
pub fn btree_to_map_with_path<V: Unpack + Copy>(
path: &mut Vec<u64>,
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
ignore_non_fatal: bool,
root: u64,
) -> Result<BTreeMap<u64, (Vec<u64>, V)>> {
let walker = BTreeWalker::new_with_sm(engine, sm, ignore_non_fatal)?;
let visitor = ValuePathCollector::<V>::new();
walker.walk(path, &visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}
//------------------------------------------

204
src/pdata/btree_builder.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{anyhow, Result};
use anyhow::Result;
use byteorder::{LittleEndian, WriteBytesExt};
use std::collections::VecDeque;
use std::io::Cursor;
@@ -9,6 +9,7 @@ use crate::io_engine::*;
use crate::pdata::btree::*;
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
use crate::write_batcher::*;
//------------------------------------------
@@ -67,24 +68,16 @@ fn calc_max_entries<V: Unpack>() -> usize {
//------------------------------------------
struct Entries<V> {
max_entries: usize,
pub max_entries: usize,
entries: VecDeque<(u64, V)>,
}
enum Action<V> {
Noop,
WriteSingle {
keys: Vec<u64>,
values: Vec<V>,
},
WritePair {
keys1: Vec<u64>,
values1: Vec<V>,
keys2: Vec<u64>,
values2: Vec<V>,
},
EmitNode(Vec<u64>, Vec<V>), // keys, values
}
use Action::*;
impl<V> Entries<V> {
pub fn new(max_entries: usize) -> Entries<V> {
Entries {
@@ -93,20 +86,19 @@ impl<V> Entries<V> {
}
}
pub fn add_entry(&mut self, k: u64, v: V) -> Action<V> {
let result = if self.full() {
pub fn add_entry(&mut self, k: u64, v: V) -> Vec<Action<V>> {
let mut result = Vec::new();
if self.full() {
let (keys, values) = self.pop(self.max_entries);
Action::WriteSingle { keys, values }
} else {
Action::Noop
};
result.push(EmitNode(keys, values));
}
self.entries.push_back((k, v));
result
}
pub fn complete(&mut self) -> Action<V> {
fn complete_(&mut self, result: &mut Vec<Action<V>>) {
let n = self.entries.len();
if n >= self.max_entries {
@@ -115,20 +107,20 @@ impl<V> Entries<V> {
let (keys1, values1) = self.pop(n1);
let (keys2, values2) = self.pop(n2);
Action::WritePair {
keys1,
values1,
keys2,
values2,
}
result.push(EmitNode(keys1, values1));
result.push(EmitNode(keys2, values2));
} else if n > 0 {
let (keys, values) = self.pop(n);
Action::WriteSingle { keys, values }
} else {
Action::Noop
result.push(EmitNode(keys, values));
}
}
pub fn complete(&mut self) -> Vec<Action<V>> {
let mut result = Vec::new();
self.complete_(&mut result);
result
}
fn full(&self) -> bool {
self.entries.len() >= 2 * self.max_entries
}
@@ -149,55 +141,11 @@ impl<V> Entries<V> {
//------------------------------------------
struct WriteBatcher {
engine: Arc<Box<dyn IoEngine>>,
sm: Arc<Mutex<dyn SpaceMap>>,
batch_size: usize,
queue: Vec<Block>,
}
impl WriteBatcher {
fn new(
engine: Arc<Box<dyn IoEngine>>,
sm: Arc<Mutex<dyn SpaceMap>>,
batch_size: usize,
) -> WriteBatcher {
WriteBatcher {
engine,
sm,
batch_size,
queue: Vec::with_capacity(batch_size),
}
}
fn alloc(&mut self) -> Result<u64> {
let mut sm = self.sm.lock().unwrap();
let b = sm.alloc()?;
if b.is_none() {
return Err(anyhow!("out of metadata space"));
}
Ok(b.unwrap())
}
fn write(&mut self, b: Block) -> Result<()> {
checksum::write_checksum(&mut b.get_data(), checksum::BT::NODE)?;
if self.queue.len() == self.batch_size {
self.flush()?;
}
self.queue.push(b);
Ok(())
}
fn flush(&mut self) -> Result<()> {
self.engine.write_many(&self.queue)?;
self.queue.clear();
Ok(())
}
pub struct NodeSummary {
block: u64,
nr_entries: usize,
key_low: u64,
key_high: u64, // inclusive
}
//------------------------------------------
@@ -208,11 +156,11 @@ fn write_node_<V: Unpack + Pack>(w: &mut WriteBatcher, mut node: Node<V>) -> Res
let loc = w.alloc()?;
node.set_block(loc);
let b = Block::new(loc);
let mut cursor = Cursor::new(b.get_data());
pack_node(&node, &mut cursor)?;
w.write(b)?;
w.write(b, checksum::BT::NODE)?;
Ok((first_key, loc))
}
@@ -268,7 +216,10 @@ pub struct Builder<V: Unpack + Pack> {
}
impl<V: Unpack + Pack> Builder<V> {
pub fn new(engine: Arc<Box<dyn IoEngine>>, sm: Arc<Mutex<dyn SpaceMap>>) -> Builder<V> {
pub fn new(
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap>>,
) -> Builder<V> {
let max_entries = calc_max_entries::<V>();
let max_internal_entries = calc_max_entries::<u64>();
@@ -282,13 +233,41 @@ impl<V: Unpack + Pack> Builder<V> {
}
pub fn add_entry(&mut self, k: u64, v: V) -> Result<()> {
let action = self.entries.add_entry(k, v);
self.perform_action(action)
let actions = self.entries.add_entry(k, v);
for a in actions {
self.perform_action(a)?;
}
Ok(())
}
pub fn add_leaf_node(&mut self, leaf: &NodeSummary) -> Result<()> {
match leaf.nr_entries {
n if n == 0 => {
// Do nothing
},
n if n < (self.entries.max_entries / 2) => {
// FIXME: what if we've already queued a handful of entries for a node?
// Add the entries individually
todo!();
},
n => {
let actions = self.entries.complete();
for a in actions {
self.perform_action(a)?;
}
self.add_internal_entry(0, leaf.key_low, leaf.block)?;
}
}
Ok(())
}
pub fn complete(mut self) -> Result<u64> {
let action = self.entries.complete();
self.perform_action(action)?;
let actions = self.entries.complete();
for a in actions {
self.perform_action(a)?;
}
self.w.flush()?;
Ok(self.root)
}
@@ -297,33 +276,26 @@ impl<V: Unpack + Pack> Builder<V> {
fn add_internal_entry(&mut self, level: usize, k: u64, v: u64) -> Result<()> {
if self.internal_entries.len() == level {
self.internal_entries.push(Entries::new(self.max_internal_entries));
self.internal_entries
.push(Entries::new(self.max_internal_entries));
}
let action = self.internal_entries[level].add_entry(k, v);
self.perform_internal_action(level, action)
let actions = self.internal_entries[level].add_entry(k, v);
for a in actions {
self.perform_internal_action(level, a)?;
}
Ok(())
}
fn perform_internal_action(&mut self, level: usize, action: Action<u64>) -> Result<()> {
match action {
Action::Noop => {}
Action::WriteSingle { keys, values } => {
EmitNode(keys, values) => {
let (k, loc) = write_internal(&mut self.w, keys, values)?;
self.add_internal_entry(level + 1, k, loc)?;
self.root = loc;
}
Action::WritePair {
keys1,
values1,
keys2,
values2,
} => {
let (k, loc) = write_leaf(&mut self.w, keys1, values1)?;
self.add_internal_entry(level + 1, k, loc)?;
let (k, loc) = write_leaf(&mut self.w, keys2, values2)?;
self.add_internal_entry(level + 1, k, loc)?;
}
},
}
Ok(())
@@ -331,23 +303,10 @@ impl<V: Unpack + Pack> Builder<V> {
fn perform_action<V2: Unpack + Pack>(&mut self, action: Action<V2>) -> Result<()> {
match action {
Action::Noop => {}
Action::WriteSingle { keys, values } => {
EmitNode(keys, values) => {
let (k, loc) = write_leaf(&mut self.w, keys, values)?;
self.add_internal_entry(0, k, loc)?;
}
Action::WritePair {
keys1,
values1,
keys2,
values2,
} => {
let (k, loc) = write_leaf(&mut self.w, keys1, values1)?;
self.add_internal_entry(0, k, loc)?;
let (k, loc) = write_leaf(&mut self.w, keys2, values2)?;
self.add_internal_entry(0, k, loc)?;
}
},
}
Ok(())
@@ -355,10 +314,3 @@ impl<V: Unpack + Pack> Builder<V> {
}
//------------------------------------------
#[test]
fn fail() {
assert!(false);
}
//------------------------------------------

245
src/pdata/btree_leaf_walker.rs Normal file
View File

@@ -0,0 +1,245 @@
use fixedbitset::FixedBitSet;
use std::sync::{Arc, Mutex};
use crate::checksum;
use crate::io_engine::*;
use crate::pdata::btree::*;
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
//------------------------------------------
pub trait LeafVisitor<V: Unpack> {
fn visit(&mut self, kr: &KeyRange, b: u64) -> Result<()>;
// Nodes may be shared and thus visited multiple times. The walker avoids
// doing repeated IO, but it does call this method to keep the visitor up to
// date. b may be an internal node obviously.
fn visit_again(&mut self, b: u64) -> Result<()>;
fn end_walk(&mut self) -> Result<()>;
}
// This is useful if you just want to get the space map counts from the walk.
pub struct NoopLeafVisitor {}
impl<V: Unpack> LeafVisitor<V> for NoopLeafVisitor {
fn visit(&mut self, kr: &KeyRange, b: u64) -> Result<()> {
Ok(())
}
fn visit_again(&mut self, b: u64) -> Result<()> {
Ok(())
}
fn end_walk(&mut self) -> Result<()> {
Ok(())
}
}
pub struct LeafWalker<'a> {
engine: Arc<dyn IoEngine + Send + Sync>,
sm: &'a mut dyn SpaceMap,
leaves: FixedBitSet,
ignore_non_fatal: bool,
}
impl<'a> LeafWalker<'a> {
pub fn new(
engine: Arc<dyn IoEngine + Send + Sync>,
sm: &'a mut dyn SpaceMap,
ignore_non_fatal: bool,
) -> LeafWalker<'a> {
let nr_blocks = engine.get_nr_blocks() as usize;
LeafWalker {
engine,
sm,
leaves: FixedBitSet::with_capacity(nr_blocks),
ignore_non_fatal,
}
}
// Atomically increments the ref count, and returns the _old_ count.
fn sm_inc(&mut self, b: u64) -> u32 {
let sm = &mut self.sm;
let count = sm.get(b).unwrap();
sm.inc(b, 1).unwrap();
count
}
fn walk_nodes<LV, V>(
&mut self,
depth: usize,
path: &mut Vec<u64>,
visitor: &mut LV,
krs: &[KeyRange],
bs: &[u64],
) -> Result<()>
where
LV: LeafVisitor<V>,
V: Unpack,
{
assert_eq!(krs.len(), bs.len());
let mut errs: Vec<BTreeError> = Vec::new();
let mut blocks = Vec::with_capacity(bs.len());
let mut filtered_krs = Vec::with_capacity(krs.len());
for i in 0..bs.len() {
if self.sm_inc(bs[i]) == 0 {
// Node not yet seen
blocks.push(bs[i]);
filtered_krs.push(krs[i].clone());
} else {
// This node has already been checked ...
if let Err(e) = visitor.visit_again(bs[i]) {
// ... but the visitor isn't happy
errs.push(e.clone());
}
}
}
let rblocks = self
.engine
.read_many(&blocks[0..])
.map_err(|_e| io_err(path))?;
let mut i = 0;
for rb in rblocks {
match rb {
Err(_) => {
return Err(io_err(path).keys_context(&filtered_krs[i]));
}
Ok(b) => {
self.walk_node(depth - 1, path, visitor, &filtered_krs[i], &b, false)?;
}
}
i += 1;
}
Ok(())
}
fn walk_node_<LV, V>(
&mut self,
depth: usize,
path: &mut Vec<u64>,
visitor: &mut LV,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
LV: LeafVisitor<V>,
V: Unpack,
{
use Node::*;
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(node_err_s(
path,
format!("checksum failed for node {}, {:?}", b.loc, bt),
)
.keys_context(kr));
}
let node = unpack_node::<V>(path, &b.get_data(), self.ignore_non_fatal, is_root)?;
if let Internal { keys, values, .. } = node {
let krs = split_key_ranges(path, &kr, &keys)?;
if depth == 0 {
for i in 0..krs.len() {
self.sm.inc(values[i], 1).expect("sm.inc() failed");
for v in &values {
self.leaves.insert(*v as usize);
}
visitor.visit(&krs[i], values[i])?;
}
Ok(())
} else {
self.walk_nodes(depth, path, visitor, &krs, &values)
}
} else {
Err(node_err(path, "btree nodes are not all at the same depth."))
}
}
fn walk_node<LV, V>(
&mut self,
depth: usize,
path: &mut Vec<u64>,
visitor: &mut LV,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
LV: LeafVisitor<V>,
V: Unpack,
{
path.push(b.loc);
let r = self.walk_node_(depth, path, visitor, kr, b, is_root);
path.pop();
visitor.end_walk()?;
r
}
fn get_depth<V: Unpack>(&self, path: &mut Vec<u64>, root: u64, is_root: bool) -> Result<usize> {
use Node::*;
let b = self.engine.read(root).map_err(|_| io_err(path))?;
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(node_err_s(
path,
format!("checksum failed for node {}, {:?}", root, bt),
));
}
let node = unpack_node::<V>(path, &b.get_data(), self.ignore_non_fatal, is_root)?;
match node {
Internal { values, .. } => {
let n = self.get_depth::<V>(path, values[0], false)?;
Ok(n + 1)
}
Leaf { .. } => Ok(0),
}
}
pub fn walk<LV, V>(&mut self, path: &mut Vec<u64>, visitor: &mut LV, root: u64) -> Result<()>
where
LV: LeafVisitor<V>,
V: Unpack,
{
let kr = KeyRange {
start: None,
end: None,
};
let depth = self.get_depth::<V>(path, root, true)?;
if depth == 0 {
self.sm_inc(root);
self.leaves.insert(root as usize);
visitor.visit(&kr, root)?;
Ok(())
} else {
if self.sm_inc(root) > 0 {
visitor.visit_again(root)
} else {
let root = self.engine.read(root).map_err(|_| io_err(path))?;
self.walk_node(depth - 1, path, visitor, &kr, &root, true)
}
}
}
// Call this to extract the leaves bitset after you've done your walking.
pub fn get_leaves(self) -> FixedBitSet {
self.leaves
}
}
//------------------------------------------

136
src/pdata/btree_merge.rs Normal file
View File

@@ -0,0 +1,136 @@
use anyhow::{anyhow, Result};
use byteorder::{LittleEndian, WriteBytesExt};
use std::collections::VecDeque;
use std::io::Cursor;
use std::sync::{Arc, Mutex};
use crate::checksum;
use crate::io_engine::*;
use crate::pdata::btree;
use crate::pdata::btree::*;
use crate::pdata::btree_walker::*;
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
use crate::write_batcher::*;
//------------------------------------------
// The subtrees will often consist of a single under populated leaf node. Given this
// we're going to merge by:
// i) Building an ordered list of all leaf nodes across all subtrees.
// ii) Merge leaf nodes where they can be packed more efficiently (non destructively to original subtrees).
// iii) Build higher levels from scratch. There are very few of these internal nodes compared to leaves anyway.
struct NodeSummary {
block: u64,
nr_entries: usize,
key_low: u64,
key_high: u64, // inclusive
}
struct LVInner {
last_key: Option<u64>,
leaves: Vec<NodeSummary>,
}
struct LeafVisitor {
inner: Mutex<LVInner>,
}
impl LeafVisitor {
fn new() -> LeafVisitor {
LeafVisitor {
inner: Mutex::new(LVInner {
last_key: None,
leaves: Vec::new(),
}),
}
}
}
impl<V: Unpack> NodeVisitor<V> for LeafVisitor {
fn visit(
&self,
path: &Vec<u64>,
kr: &KeyRange,
header: &NodeHeader,
keys: &[u64],
values: &[V],
) -> btree::Result<()> {
// ignore empty nodes
if keys.len() == 0 {
return Ok(());
}
let mut inner = self.inner.lock().unwrap();
// Check keys are ordered.
if inner.leaves.len() > 0 {
let last_key = inner.leaves.last().unwrap().key_high;
if keys[0] <= last_key {
return Err(BTreeError::NodeError(
"unable to merge btrees: sub trees out of order".to_string(),
));
}
}
let l = NodeSummary {
block: *path.last().unwrap(),
nr_entries: keys.len(),
key_low: keys[0],
key_high: *keys.last().unwrap(),
};
inner.leaves.push(l);
Ok(())
}
fn visit_again(&self, path: &Vec<u64>, b: u64) -> btree::Result<()> {
Ok(())
}
fn end_walk(&self) -> btree::Result<()> {
Ok(())
}
}
pub type AEngine = Arc<dyn IoEngine + Send + Sync>;
fn collect_leaves<V: Unpack>(engine: AEngine, roots: &[u64]) -> Result<Vec<NodeSummary>> {
let lv = LeafVisitor::new();
let walker = BTreeWalker::new(engine, false);
let mut path = Vec::new();
for root in roots {
walker.walk::<LeafVisitor, V>(&mut path, &lv, *root)?;
}
Ok(lv.inner.into_inner().unwrap().leaves)
}
//------------------------------------------
fn optimise_leaves<V: Unpack + Pack>(
batcher: &mut WriteBatcher,
lvs: Vec<NodeSummary>,
) -> Result<Vec<NodeSummary>> {
// FIXME: implement
Ok(lvs)
}
//------------------------------------------
pub fn merge<V: Unpack + Pack>(
engine: AEngine,
sm: Arc<Mutex<dyn SpaceMap>>,
roots: &[u64],
) -> Result<u64> {
let lvs = collect_leaves::<V>(engine.clone(), roots)?;
let mut batcher = WriteBatcher::new(engine, sm, 256);
let lvs = optimise_leaves::<V>(&mut batcher, lvs)?;
todo!();
}
//------------------------------------------

573
src/pdata/btree_walker.rs Normal file
View File

@@ -0,0 +1,573 @@
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use threadpool::ThreadPool;
use crate::checksum;
use crate::io_engine::*;
use crate::pdata::btree::*;
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
//------------------------------------------
pub trait NodeVisitor<V: Unpack> {
// &self is deliberately non mut to allow the walker to use multiple threads.
fn visit(
&self,
path: &Vec<u64>,
kr: &KeyRange,
header: &NodeHeader,
keys: &[u64],
values: &[V],
) -> Result<()>;
// Nodes may be shared and thus visited multiple times. The walker avoids
// doing repeated IO, but it does call this method to keep the visitor up to
// date.
fn visit_again(&self, path: &Vec<u64>, b: u64) -> Result<()>;
fn end_walk(&self) -> Result<()>;
}
#[derive(Clone)]
pub struct BTreeWalker {
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
fails: Arc<Mutex<BTreeMap<u64, BTreeError>>>,
ignore_non_fatal: bool,
}
impl BTreeWalker {
pub fn new(engine: Arc<dyn IoEngine + Send + Sync>, ignore_non_fatal: bool) -> BTreeWalker {
let nr_blocks = engine.get_nr_blocks() as usize;
let r: BTreeWalker = BTreeWalker {
engine,
sm: Arc::new(Mutex::new(RestrictedSpaceMap::new(nr_blocks as u64))),
fails: Arc::new(Mutex::new(BTreeMap::new())),
ignore_non_fatal,
};
r
}
pub fn new_with_sm(
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
ignore_non_fatal: bool,
) -> Result<BTreeWalker> {
{
let sm = sm.lock().unwrap();
assert_eq!(sm.get_nr_blocks().unwrap(), engine.get_nr_blocks());
}
Ok(BTreeWalker {
engine,
sm,
fails: Arc::new(Mutex::new(BTreeMap::new())),
ignore_non_fatal,
})
}
fn failed(&self, b: u64) -> Option<BTreeError> {
let fails = self.fails.lock().unwrap();
match fails.get(&b) {
None => None,
Some(e) => Some(e.clone()),
}
}
fn set_fail(&self, b: u64, err: BTreeError) {
// FIXME: should we monitor the size of fails, and abort if too many errors?
let mut fails = self.fails.lock().unwrap();
fails.insert(b, err);
}
// Atomically increments the ref count, and returns the _old_ count.
fn sm_inc(&self, b: u64) -> u32 {
let mut sm = self.sm.lock().unwrap();
let count = sm.get(b).unwrap();
sm.inc(b, 1).unwrap();
count
}
fn build_aggregate(&self, b: u64, errs: Vec<BTreeError>) -> Result<()> {
match errs.len() {
0 => Ok(()),
1 => {
let e = errs[0].clone();
self.set_fail(b, e.clone());
Err(e)
}
_ => {
let e = aggregate_error(errs);
self.set_fail(b, e.clone());
Err(e)
}
}
}
fn walk_nodes<NV, V>(
&self,
path: &mut Vec<u64>,
visitor: &NV,
krs: &[KeyRange],
bs: &[u64],
) -> Vec<BTreeError>
where
NV: NodeVisitor<V>,
V: Unpack,
{
assert_eq!(krs.len(), bs.len());
let mut errs: Vec<BTreeError> = Vec::new();
let mut blocks = Vec::with_capacity(bs.len());
let mut filtered_krs = Vec::with_capacity(krs.len());
for i in 0..bs.len() {
if self.sm_inc(bs[i]) == 0 {
// Node not yet seen
blocks.push(bs[i]);
filtered_krs.push(krs[i].clone());
} else {
// This node has already been checked ...
match self.failed(bs[i]) {
None => {
// ... it was clean.
if let Err(e) = visitor.visit_again(path, bs[i]) {
// ... but the visitor isn't happy
errs.push(e.clone());
}
}
Some(e) => {
// ... there was an error
errs.push(e.clone());
}
}
}
}
match self.engine.read_many(&blocks[0..]) {
Err(_) => {
// IO completely failed, error every block
for (i, b) in blocks.iter().enumerate() {
let e = io_err(path).keys_context(&filtered_krs[i]);
errs.push(e.clone());
self.set_fail(*b, e);
}
}
Ok(rblocks) => {
let mut i = 0;
for rb in rblocks {
match rb {
Err(_) => {
let e = io_err(path).keys_context(&filtered_krs[i]);
errs.push(e.clone());
self.set_fail(blocks[i], e);
}
Ok(b) => match self.walk_node(path, visitor, &filtered_krs[i], &b, false) {
Err(e) => {
errs.push(e);
}
Ok(()) => {}
},
}
i += 1;
}
}
}
errs
}
fn walk_node_<NV, V>(
&self,
path: &mut Vec<u64>,
visitor: &NV,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
use Node::*;
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(node_err_s(
path,
format!("checksum failed for node {}, {:?}", b.loc, bt),
)
.keys_context(kr));
}
let node = unpack_node::<V>(path, &b.get_data(), self.ignore_non_fatal, is_root)?;
match node {
Internal { keys, values, .. } => {
let krs = split_key_ranges(path, &kr, &keys)?;
let errs = self.walk_nodes(path, visitor, &krs, &values);
return self.build_aggregate(b.loc, errs);
}
Leaf {
header,
keys,
values,
} => {
if let Err(e) = visitor.visit(path, &kr, &header, &keys, &values) {
let e = BTreeError::Path(path.clone(), Box::new(e.clone()));
self.set_fail(b.loc, e.clone());
return Err(e);
}
}
}
Ok(())
}
fn walk_node<NV, V>(
&self,
path: &mut Vec<u64>,
visitor: &NV,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
path.push(b.loc);
let r = self.walk_node_(path, visitor, kr, b, is_root);
path.pop();
visitor.end_walk()?;
r
}
pub fn walk<NV, V>(&self, path: &mut Vec<u64>, visitor: &NV, root: u64) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
if self.sm_inc(root) > 0 {
if let Some(e) = self.failed(root) {
Err(e.clone())
} else {
visitor.visit_again(path, root)
}
} else {
let root = self.engine.read(root).map_err(|_| io_err(path))?;
let kr = KeyRange {
start: None,
end: None,
};
self.walk_node(path, visitor, &kr, &root, true)
}
}
}
//--------------------------------
fn walk_node_threaded_<NV, V>(
w: Arc<BTreeWalker>,
path: &mut Vec<u64>,
pool: &ThreadPool,
visitor: Arc<NV>,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
use Node::*;
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(node_err_s(
path,
format!("checksum failed for node {}, {:?}", b.loc, bt),
)
.keys_context(kr));
}
let node = unpack_node::<V>(path, &b.get_data(), w.ignore_non_fatal, is_root)?;
match node {
Internal { keys, values, .. } => {
let krs = split_key_ranges(path, &kr, &keys)?;
let errs = walk_nodes_threaded(w.clone(), path, pool, visitor, &krs, &values);
return w.build_aggregate(b.loc, errs);
}
Leaf {
header,
keys,
values,
} => {
visitor.visit(path, kr, &header, &keys, &values)?;
}
}
Ok(())
}
fn walk_node_threaded<NV, V>(
w: Arc<BTreeWalker>,
path: &mut Vec<u64>,
pool: &ThreadPool,
visitor: Arc<NV>,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
path.push(b.loc);
let r = walk_node_threaded_(w, path, pool, visitor.clone(), kr, b, is_root);
path.pop();
visitor.end_walk()?;
r
}
fn walk_nodes_threaded<NV, V>(
w: Arc<BTreeWalker>,
path: &mut Vec<u64>,
pool: &ThreadPool,
visitor: Arc<NV>,
krs: &[KeyRange],
bs: &[u64],
) -> Vec<BTreeError>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
assert_eq!(krs.len(), bs.len());
let mut errs: Vec<BTreeError> = Vec::new();
let mut blocks = Vec::with_capacity(bs.len());
let mut filtered_krs = Vec::with_capacity(krs.len());
for i in 0..bs.len() {
if w.sm_inc(bs[i]) == 0 {
// Node not yet seen
blocks.push(bs[i]);
filtered_krs.push(krs[i].clone());
} else {
// This node has already been checked ...
match w.failed(bs[i]) {
None => {
// ... it was clean.
if let Err(e) = visitor.visit_again(path, bs[i]) {
// ... but the visitor isn't happy
errs.push(e.clone());
}
}
Some(e) => {
// ... there was an error
errs.push(e.clone());
}
}
}
}
match w.engine.read_many(&blocks[0..]) {
Err(_) => {
// IO completely failed error every block
for (i, b) in blocks.iter().enumerate() {
let e = io_err(path).keys_context(&filtered_krs[i]);
errs.push(e.clone());
w.set_fail(*b, e);
}
}
Ok(rblocks) => {
let mut i = 0;
let errs = Arc::new(Mutex::new(Vec::new()));
for rb in rblocks {
match rb {
Err(_) => {
let e = io_err(path).keys_context(&filtered_krs[i]);
let mut errs = errs.lock().unwrap();
errs.push(e.clone());
w.set_fail(blocks[i], e);
}
Ok(b) => {
let w = w.clone();
let visitor = visitor.clone();
let kr = filtered_krs[i].clone();
let errs = errs.clone();
let mut path = path.clone();
pool.execute(move || {
match w.walk_node(&mut path, visitor.as_ref(), &kr, &b, false) {
Err(e) => {
let mut errs = errs.lock().unwrap();
errs.push(e);
}
Ok(()) => {}
}
});
}
}
i += 1;
}
pool.join();
}
}
errs
}
pub fn walk_threaded<NV, V>(
path: &mut Vec<u64>,
w: Arc<BTreeWalker>,
pool: &ThreadPool,
visitor: Arc<NV>,
root: u64,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
if w.sm_inc(root) > 0 {
if let Some(e) = w.failed(root) {
Err(e.clone())
} else {
visitor.visit_again(path, root)
}
} else {
let root = w.engine.read(root).map_err(|_| io_err(path))?;
let kr = KeyRange {
start: None,
end: None,
};
walk_node_threaded(w, path, pool, visitor, &kr, &root, true)
}
}
//------------------------------------------
struct ValueCollector<V> {
values: Mutex<BTreeMap<u64, V>>,
}
impl<V> ValueCollector<V> {
fn new() -> ValueCollector<V> {
ValueCollector {
values: Mutex::new(BTreeMap::new()),
}
}
}
// FIXME: should we be using Copy rather than clone? (Yes)
impl<V: Unpack + Copy> NodeVisitor<V> for ValueCollector<V> {
fn visit(
&self,
_path: &Vec<u64>,
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
values: &[V],
) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], values[n].clone());
}
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> Result<()> {
Ok(())
}
fn end_walk(&self) -> Result<()> {
Ok(())
}
}
pub fn btree_to_map<V: Unpack + Copy>(
path: &mut Vec<u64>,
engine: Arc<dyn IoEngine + Send + Sync>,
ignore_non_fatal: bool,
root: u64,
) -> Result<BTreeMap<u64, V>> {
let walker = BTreeWalker::new(engine, ignore_non_fatal);
let visitor = ValueCollector::<V>::new();
walker.walk(path, &visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}
pub fn btree_to_map_with_sm<V: Unpack + Copy>(
path: &mut Vec<u64>,
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
ignore_non_fatal: bool,
root: u64,
) -> Result<BTreeMap<u64, V>> {
let walker = BTreeWalker::new_with_sm(engine, sm, ignore_non_fatal)?;
let visitor = ValueCollector::<V>::new();
walker.walk(path, &visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}
//------------------------------------------
struct ValuePathCollector<V> {
values: Mutex<BTreeMap<u64, (Vec<u64>, V)>>,
}
impl<V> ValuePathCollector<V> {
fn new() -> ValuePathCollector<V> {
ValuePathCollector {
values: Mutex::new(BTreeMap::new()),
}
}
}
impl<V: Unpack + Clone> NodeVisitor<V> for ValuePathCollector<V> {
fn visit(
&self,
path: &Vec<u64>,
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
values: &[V],
) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], (path.clone(), values[n].clone()));
}
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> Result<()> {
Ok(())
}
fn end_walk(&self) -> Result<()> {
Ok(())
}
}
pub fn btree_to_map_with_path<V: Unpack + Copy>(
path: &mut Vec<u64>,
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
ignore_non_fatal: bool,
root: u64,
) -> Result<BTreeMap<u64, (Vec<u64>, V)>> {
let walker = BTreeWalker::new_with_sm(engine, sm, ignore_non_fatal)?;
let visitor = ValuePathCollector::<V>::new();
walker.walk(path, &visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}
//------------------------------------------

3
src/pdata/mod.rs
View File

@@ -1,5 +1,8 @@
pub mod btree;
pub mod btree_builder;
pub mod btree_merge;
pub mod btree_leaf_walker;
pub mod btree_walker;
pub mod space_map;
pub mod unpack;

11
src/pdata/space_map.rs
View File

@@ -3,6 +3,7 @@ use byteorder::{LittleEndian, WriteBytesExt};
use fixedbitset::FixedBitSet;
use nom::{multi::count, number::complete::*, IResult};
use std::sync::{Arc, Mutex};
use std::boxed::Box;
use crate::io_engine::*;
use crate::pdata::unpack::{Pack, Unpack};
@@ -328,6 +329,16 @@ pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc<Mutex<dyn SpaceMap + Send
}
}
pub fn core_sm_without_mutex(nr_entries: u64, max_count: u32) -> Box<dyn SpaceMap> {
if max_count <= u8::MAX as u32 {
Box::new(CoreSpaceMap::<u8>::new(nr_entries))
} else if max_count <= u16::MAX as u32 {
Box::new(CoreSpaceMap::<u16>::new(nr_entries))
} else {
Box::new(CoreSpaceMap::<u32>::new(nr_entries))
}
}
//------------------------------------------
// This in core space map can only count to one, useful when walking