[era_check] Now checks bloom tree
This commit is contained in:
Joe Thornber
parent
41d25aa9e4
commit
bed0f369a8
|
|
@ -51,6 +51,7 @@ SOURCE=\
|
|||
\
|
||||
era/era_detail.cc \
|
||||
era/superblock.cc \
|
||||
era/bloom_tree.cc \
|
||||
\
|
||||
persistent-data/checksum.cc \
|
||||
persistent-data/error_set.cc \
|
||||
|
|
@ -299,6 +300,7 @@ ERA_CHECK_SOURCE=\
|
|||
base/error_state.cc \
|
||||
base/endian_utils.cc \
|
||||
\
|
||||
era/bloom_tree.cc \
|
||||
era/superblock.cc \
|
||||
era/era_detail.cc \
|
||||
\
|
||||
|
|
|
|||
|
|
@ -0,0 +1,126 @@
|
|||
#include "era/bloom_tree.h"
|
||||
#include "persistent-data/data-structures/btree_damage_visitor.h"
|
||||
#include "persistent-data/data-structures/bitset.h"
|
||||
|
||||
using namespace boost;
|
||||
using namespace era;
|
||||
using namespace bloom_tree_detail;
|
||||
using namespace persistent_data;
|
||||
using namespace std;
|
||||
|
||||
//----------------------------------------------------------------
|
||||
|
||||
missing_eras::missing_eras(string const &desc,
|
||||
run<uint32_t> const &eras)
|
||||
: damage(desc),
|
||||
eras_(eras)
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
missing_eras::visit(damage_visitor &v) const {
|
||||
v.visit(*this);
|
||||
}
|
||||
|
||||
damaged_bloom_filter::damaged_bloom_filter(string const &desc,
|
||||
uint32_t era,
|
||||
run<uint32_t> missing_bits)
|
||||
: damage(desc),
|
||||
era_(era),
|
||||
missing_bits_(missing_bits)
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
damaged_bloom_filter::visit(damage_visitor &v) const
|
||||
{
|
||||
v.visit(*this);
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------
|
||||
|
||||
namespace {
|
||||
class ll_bloom_visitor : public bitset_detail::bitset_visitor {
|
||||
public:
|
||||
typedef persistent_data::transaction_manager::ptr tm_ptr;
|
||||
|
||||
ll_bloom_visitor(tm_ptr tm,
|
||||
bloom_tree_detail::bloom_visitor &bloom_v,
|
||||
bloom_tree_detail::damage_visitor &dv)
|
||||
: tm_(tm),
|
||||
bloom_v_(bloom_v),
|
||||
dv_(dv) {
|
||||
}
|
||||
|
||||
void visit(btree_path const &path, era_detail const &era) {
|
||||
era_ = path[0];
|
||||
bitset bs(tm_, era.bloom_root, era.nr_bits);
|
||||
bs.walk_bitset(*this);
|
||||
}
|
||||
|
||||
void visit(uint32_t index, bool value) {
|
||||
bloom_v_.visit(index, value);
|
||||
}
|
||||
|
||||
void visit(bitset_detail::missing_bits const &d) {
|
||||
dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_));
|
||||
}
|
||||
|
||||
private:
|
||||
tm_ptr tm_;
|
||||
uint64_t era_;
|
||||
bloom_tree_detail::bloom_visitor &bloom_v_;
|
||||
bloom_tree_detail::damage_visitor &dv_;
|
||||
};
|
||||
|
||||
class ll_damage_visitor {
|
||||
public:
|
||||
ll_damage_visitor(damage_visitor &v)
|
||||
: v_(v) {
|
||||
}
|
||||
|
||||
virtual void visit(btree_path const &path,
|
||||
btree_detail::damage const &d) {
|
||||
v_.visit(missing_eras(d.desc_, to_uint32(d.lost_keys_)));
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
run<uint32_t> to_uint32(run<T> const &r) {
|
||||
return run<uint32_t>(optional<uint32_t>(r.begin_),
|
||||
optional<uint32_t>(r.end_));
|
||||
}
|
||||
|
||||
damage_visitor &v_;
|
||||
};
|
||||
}
|
||||
|
||||
void
|
||||
era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
||||
bloom_tree const &tree,
|
||||
bloom_tree_detail::bloom_visitor &bloom_v,
|
||||
bloom_tree_detail::damage_visitor &dv)
|
||||
{
|
||||
ll_bloom_visitor ll_bv(tm, bloom_v, dv);
|
||||
ll_damage_visitor ll_dv(dv);
|
||||
btree_visit_values(tree, ll_bv, ll_dv);
|
||||
}
|
||||
|
||||
namespace {
|
||||
class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor {
|
||||
public:
|
||||
void visit(uint32_t index, bool value) {
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
void
|
||||
era::check_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
||||
bloom_tree const &tree,
|
||||
bloom_tree_detail::damage_visitor &dv)
|
||||
{
|
||||
noop_bloom_visitor bv;
|
||||
walk_bloom_tree(tm, tree, bv, dv);
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------
|
||||
174
era/bloom_tree.h
174
era/bloom_tree.h
|
|
@ -1,112 +1,84 @@
|
|||
#ifndef ERA_BLOOM_TREE_H
|
||||
#define ERA_BLOOM_TREE_H
|
||||
|
||||
#include "era/era_detail.h"
|
||||
#include "persistent-data/data-structures/btree.h"
|
||||
|
||||
//----------------------------------------------------------------
|
||||
|
||||
namespace era {
|
||||
namespace bloom_tree_detail {
|
||||
class damage_visitor;
|
||||
|
||||
class damage {
|
||||
public:
|
||||
damage(std::string const &desc)
|
||||
: desc_(desc) {
|
||||
}
|
||||
|
||||
virtual ~damage() {}
|
||||
virtual void visit(damage_visitor &v) const = 0;
|
||||
|
||||
std::string const &get_desc() const {
|
||||
return desc_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string desc_;
|
||||
};
|
||||
|
||||
struct missing_eras : public damage {
|
||||
missing_eras(std::string const &desc, run<uint32_t> const &eras);
|
||||
virtual void visit(damage_visitor &v) const;
|
||||
|
||||
run<uint32_t> eras_;
|
||||
};
|
||||
|
||||
struct damaged_bloom_filter : public damage {
|
||||
damaged_bloom_filter(std::string const &desc,
|
||||
uint32_t era,
|
||||
run<uint32_t> missing_bits);
|
||||
virtual void visit(damage_visitor &v) const;
|
||||
|
||||
uint32_t era_;
|
||||
run<uint32_t> missing_bits_;
|
||||
};
|
||||
|
||||
class damage_visitor {
|
||||
public:
|
||||
typedef boost::shared_ptr<damage_visitor> ptr;
|
||||
|
||||
virtual ~damage_visitor() {}
|
||||
|
||||
void visit(damage const &d) {
|
||||
d.visit(*this);
|
||||
}
|
||||
|
||||
virtual void visit(missing_eras const &d) = 0;
|
||||
virtual void visit(damaged_bloom_filter const &d) = 0;
|
||||
};
|
||||
|
||||
class bloom_visitor {
|
||||
public:
|
||||
typedef boost::shared_ptr<bloom_visitor> ptr;
|
||||
|
||||
virtual ~bloom_visitor() {}
|
||||
virtual void visit(uint32_t index, bool value) = 0;
|
||||
};
|
||||
}
|
||||
|
||||
typedef persistent_data::btree<1, era_detail_traits> bloom_tree;
|
||||
|
||||
void walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
||||
bloom_tree const &tree,
|
||||
bloom_tree_detail::bloom_visitor &bloom_v,
|
||||
bloom_tree_detail::damage_visitor &dv);
|
||||
|
||||
void check_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
||||
bloom_tree const &tree,
|
||||
bloom_tree_detail::damage_visitor &dv);
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
#if 0
|
||||
class dm_era {
|
||||
public:
|
||||
dm_era(block_address nr_blocks)
|
||||
: nr_blocks_(nr_blocks),
|
||||
era_base_(0),
|
||||
base_(nr_blocks, false) {
|
||||
}
|
||||
|
||||
set<block_address> blocks_written_since(unsigned era) const {
|
||||
|
||||
}
|
||||
|
||||
unsigned get_era() const {
|
||||
return era_base_ + eras_.size() - 1;
|
||||
}
|
||||
|
||||
void record_write(block_address b) {
|
||||
current_era.record_write(b);
|
||||
}
|
||||
|
||||
void resize(block_address new_size) {
|
||||
nr_blocks_ = new_size;
|
||||
push_era();
|
||||
base_.resize(new_size, false);
|
||||
}
|
||||
|
||||
private:
|
||||
era_details ¤t_era() {
|
||||
return eras_.back();
|
||||
}
|
||||
|
||||
void need_new_era() {
|
||||
// ???
|
||||
}
|
||||
|
||||
void push_era() {
|
||||
eras_.push_back(era(nr_blocks_));
|
||||
if (eras_.size() > 100)
|
||||
pop_era();
|
||||
}
|
||||
|
||||
void pop_era() {
|
||||
era_base_++;
|
||||
|
||||
|
||||
|
||||
eras_.pop_front();
|
||||
}
|
||||
|
||||
static const unsigned NR_PROBES = 6;
|
||||
|
||||
class era_details {
|
||||
public:
|
||||
era_details(block_address nr_blocks)
|
||||
: nr_blocks_(nr_blocks),
|
||||
f(power_bits(nr_blocks, NR_PROBES)) {
|
||||
}
|
||||
|
||||
void record_write(block_address b) {
|
||||
f.add(b);
|
||||
}
|
||||
|
||||
void add_blocks_written(set<block_address &result) const {
|
||||
for (block_address b = 0; b < nr_blocks; b++)
|
||||
if (f.test(b))
|
||||
result.insert(b);
|
||||
}
|
||||
|
||||
private:
|
||||
static unsigned power_bits(block_address nr_blocks) {
|
||||
// We're expecting 1% of the cache to change per era
|
||||
block_address expected_writes = nr_blocks / 100;
|
||||
|
||||
unsigned r = 1;
|
||||
while ((1ull << r) < (16 * expected_writes))
|
||||
r++;
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
typedef bloom_filter<block_address_bloom_traits> filter;
|
||||
|
||||
block_address nr_blocks;
|
||||
filter f;
|
||||
};
|
||||
|
||||
space_map::ptr setup_core_map() {
|
||||
space_map::ptr sm(new core_map(NR_BLOCKS));
|
||||
sm->inc(SUPERBLOCK);
|
||||
return sm;
|
||||
}
|
||||
|
||||
block_address nr_blocks_;
|
||||
unsigned era_base_;
|
||||
vector<bool> base_;
|
||||
deque<era_details> eras_;
|
||||
};
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
#include "base/error_state.h"
|
||||
#include "base/nested_output.h"
|
||||
#include "era/bloom_tree.h"
|
||||
#include "era/superblock.h"
|
||||
#include "persistent-data/block.h"
|
||||
#include "persistent-data/file_utils.h"
|
||||
|
|
@ -90,6 +91,40 @@ namespace {
|
|||
using reporter_base::get_error;
|
||||
};
|
||||
|
||||
class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base {
|
||||
public:
|
||||
bloom_tree_reporter(nested_output &o)
|
||||
: reporter_base(o) {
|
||||
}
|
||||
|
||||
void visit(bloom_tree_detail::missing_eras const &d) {
|
||||
out() << "missing eras" << end_message();
|
||||
{
|
||||
nested_output::nest _ = push();
|
||||
out() << d.get_desc() << end_message();
|
||||
out() << "Effected eras: [" << d.eras_.begin_.get()
|
||||
<< ", " << d.eras_.end_.get() << ")" << end_message();
|
||||
}
|
||||
|
||||
mplus_error(FATAL);
|
||||
}
|
||||
|
||||
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
|
||||
out() << "damaged bloom filter" << end_message();
|
||||
{
|
||||
nested_output::nest _ = push();
|
||||
out() << d.get_desc() << end_message();
|
||||
out() << "Era: " << d.era_ << end_message();
|
||||
out() << "Missing bits: [" << d.missing_bits_.begin_.get()
|
||||
<< ", " << d.missing_bits_.end_.get() << ")" << end_message();
|
||||
}
|
||||
|
||||
mplus_error(FATAL);
|
||||
}
|
||||
|
||||
using reporter_base::get_error;
|
||||
};
|
||||
|
||||
//--------------------------------
|
||||
|
||||
transaction_manager::ptr open_tm(block_manager<>::ptr bm) {
|
||||
|
|
@ -145,16 +180,16 @@ namespace {
|
|||
|
||||
return sb_rep.get_error();
|
||||
|
||||
#if 0
|
||||
superblock sb = read_superblock(bm);
|
||||
transaction_manager::ptr tm = open_tm(bm);
|
||||
|
||||
// FIXME: make an error class that's an instance of mplus
|
||||
bloom_tree_reporter bt_rep(out);
|
||||
era_detail_traits::ref_counter rc(tm);
|
||||
bloom_tree bt(tm, rc);
|
||||
check_bloom_tree(tm, bt, bt_rep);
|
||||
|
||||
return combine_errors(sb_rep.get_error(),
|
||||
combine_errors(mapping_rep.get_error(),
|
||||
combine_errors(hint_rep.get_error(),
|
||||
discard_rep.get_error())));
|
||||
#endif
|
||||
bt_rep.get_error());
|
||||
}
|
||||
|
||||
int check(string const &path, flags const &fs) {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#define ERA_DETAIL_H
|
||||
|
||||
#include "base/endian_utils.h"
|
||||
#include "persistent-data/transaction_manager.h"
|
||||
|
||||
//----------------------------------------------------------------
|
||||
|
||||
|
|
@ -25,9 +26,17 @@ namespace era {
|
|||
uint64_t bloom_root;
|
||||
};
|
||||
|
||||
struct era_detail_ref_counter {
|
||||
era_detail_ref_counter(persistent_data::transaction_manager::ptr tm);
|
||||
|
||||
void inc(persistent_data::block_address b);
|
||||
void dec(persistent_data::block_address b);
|
||||
};
|
||||
|
||||
struct era_detail_traits {
|
||||
typedef era_detail_disk disk_type;
|
||||
typedef era_detail value_type;
|
||||
typedef era_detail_ref_counter ref_counter;
|
||||
|
||||
static void unpack(disk_type const &disk, value_type &value);
|
||||
static void pack(value_type const &value, disk_type &disk);
|
||||
|
|
|
|||
Loading…
Reference in New Issue