diff --git a/Makefile.in b/Makefile.in index 82c785f..6d086a1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -51,6 +51,7 @@ SOURCE=\ \ era/era_detail.cc \ era/superblock.cc \ + era/bloom_tree.cc \ \ persistent-data/checksum.cc \ persistent-data/error_set.cc \ @@ -299,6 +300,7 @@ ERA_CHECK_SOURCE=\ base/error_state.cc \ base/endian_utils.cc \ \ + era/bloom_tree.cc \ era/superblock.cc \ era/era_detail.cc \ \ diff --git a/era/bloom_tree.cc b/era/bloom_tree.cc new file mode 100644 index 0000000..022d447 --- /dev/null +++ b/era/bloom_tree.cc @@ -0,0 +1,126 @@ +#include "era/bloom_tree.h" +#include "persistent-data/data-structures/btree_damage_visitor.h" +#include "persistent-data/data-structures/bitset.h" + +using namespace boost; +using namespace era; +using namespace bloom_tree_detail; +using namespace persistent_data; +using namespace std; + +//---------------------------------------------------------------- + +missing_eras::missing_eras(string const &desc, + run const &eras) + : damage(desc), + eras_(eras) +{ +} + +void +missing_eras::visit(damage_visitor &v) const { + v.visit(*this); +} + +damaged_bloom_filter::damaged_bloom_filter(string const &desc, + uint32_t era, + run missing_bits) + : damage(desc), + era_(era), + missing_bits_(missing_bits) +{ +} + +void +damaged_bloom_filter::visit(damage_visitor &v) const +{ + v.visit(*this); +} + +//---------------------------------------------------------------- + +namespace { + class ll_bloom_visitor : public bitset_detail::bitset_visitor { + public: + typedef persistent_data::transaction_manager::ptr tm_ptr; + + ll_bloom_visitor(tm_ptr tm, + bloom_tree_detail::bloom_visitor &bloom_v, + bloom_tree_detail::damage_visitor &dv) + : tm_(tm), + bloom_v_(bloom_v), + dv_(dv) { + } + + void visit(btree_path const &path, era_detail const &era) { + era_ = path[0]; + bitset bs(tm_, era.bloom_root, era.nr_bits); + bs.walk_bitset(*this); + } + + void visit(uint32_t index, bool value) { + bloom_v_.visit(index, value); + } + + void visit(bitset_detail::missing_bits const &d) { + dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_)); + } + + private: + tm_ptr tm_; + uint64_t era_; + bloom_tree_detail::bloom_visitor &bloom_v_; + bloom_tree_detail::damage_visitor &dv_; + }; + + class ll_damage_visitor { + public: + ll_damage_visitor(damage_visitor &v) + : v_(v) { + } + + virtual void visit(btree_path const &path, + btree_detail::damage const &d) { + v_.visit(missing_eras(d.desc_, to_uint32(d.lost_keys_))); + } + + private: + template + run to_uint32(run const &r) { + return run(optional(r.begin_), + optional(r.end_)); + } + + damage_visitor &v_; + }; +} + +void +era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm, + bloom_tree const &tree, + bloom_tree_detail::bloom_visitor &bloom_v, + bloom_tree_detail::damage_visitor &dv) +{ + ll_bloom_visitor ll_bv(tm, bloom_v, dv); + ll_damage_visitor ll_dv(dv); + btree_visit_values(tree, ll_bv, ll_dv); +} + +namespace { + class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor { + public: + void visit(uint32_t index, bool value) { + } + }; +}; + +void +era::check_bloom_tree(persistent_data::transaction_manager::ptr tm, + bloom_tree const &tree, + bloom_tree_detail::damage_visitor &dv) +{ + noop_bloom_visitor bv; + walk_bloom_tree(tm, tree, bv, dv); +} + +//---------------------------------------------------------------- diff --git a/era/bloom_tree.h b/era/bloom_tree.h index e98ed37..8d89a96 100644 --- a/era/bloom_tree.h +++ b/era/bloom_tree.h @@ -1,112 +1,84 @@ #ifndef ERA_BLOOM_TREE_H #define ERA_BLOOM_TREE_H +#include "era/era_detail.h" +#include "persistent-data/data-structures/btree.h" + //---------------------------------------------------------------- namespace era { + namespace bloom_tree_detail { + class damage_visitor; + + class damage { + public: + damage(std::string const &desc) + : desc_(desc) { + } + + virtual ~damage() {} + virtual void visit(damage_visitor &v) const = 0; + + std::string const &get_desc() const { + return desc_; + } + + private: + std::string desc_; + }; + + struct missing_eras : public damage { + missing_eras(std::string const &desc, run const &eras); + virtual void visit(damage_visitor &v) const; + + run eras_; + }; + + struct damaged_bloom_filter : public damage { + damaged_bloom_filter(std::string const &desc, + uint32_t era, + run missing_bits); + virtual void visit(damage_visitor &v) const; + + uint32_t era_; + run missing_bits_; + }; + + class damage_visitor { + public: + typedef boost::shared_ptr ptr; + + virtual ~damage_visitor() {} + + void visit(damage const &d) { + d.visit(*this); + } + + virtual void visit(missing_eras const &d) = 0; + virtual void visit(damaged_bloom_filter const &d) = 0; + }; + + class bloom_visitor { + public: + typedef boost::shared_ptr ptr; + + virtual ~bloom_visitor() {} + virtual void visit(uint32_t index, bool value) = 0; + }; + } + + typedef persistent_data::btree<1, era_detail_traits> bloom_tree; + + void walk_bloom_tree(persistent_data::transaction_manager::ptr tm, + bloom_tree const &tree, + bloom_tree_detail::bloom_visitor &bloom_v, + bloom_tree_detail::damage_visitor &dv); + + void check_bloom_tree(persistent_data::transaction_manager::ptr tm, + bloom_tree const &tree, + bloom_tree_detail::damage_visitor &dv); } //---------------------------------------------------------------- #endif - - -#if 0 - class dm_era { - public: - dm_era(block_address nr_blocks) - : nr_blocks_(nr_blocks), - era_base_(0), - base_(nr_blocks, false) { - } - - set blocks_written_since(unsigned era) const { - - } - - unsigned get_era() const { - return era_base_ + eras_.size() - 1; - } - - void record_write(block_address b) { - current_era.record_write(b); - } - - void resize(block_address new_size) { - nr_blocks_ = new_size; - push_era(); - base_.resize(new_size, false); - } - - private: - era_details ¤t_era() { - return eras_.back(); - } - - void need_new_era() { - // ??? - } - - void push_era() { - eras_.push_back(era(nr_blocks_)); - if (eras_.size() > 100) - pop_era(); - } - - void pop_era() { - era_base_++; - - - - eras_.pop_front(); - } - - static const unsigned NR_PROBES = 6; - - class era_details { - public: - era_details(block_address nr_blocks) - : nr_blocks_(nr_blocks), - f(power_bits(nr_blocks, NR_PROBES)) { - } - - void record_write(block_address b) { - f.add(b); - } - - void add_blocks_written(set filter; - - block_address nr_blocks; - filter f; - }; - - space_map::ptr setup_core_map() { - space_map::ptr sm(new core_map(NR_BLOCKS)); - sm->inc(SUPERBLOCK); - return sm; - } - - block_address nr_blocks_; - unsigned era_base_; - vector base_; - deque eras_; - }; -#endif diff --git a/era/era_check.cc b/era/era_check.cc index 5b143f5..991da05 100644 --- a/era/era_check.cc +++ b/era/era_check.cc @@ -14,6 +14,7 @@ #include "base/error_state.h" #include "base/nested_output.h" +#include "era/bloom_tree.h" #include "era/superblock.h" #include "persistent-data/block.h" #include "persistent-data/file_utils.h" @@ -90,6 +91,40 @@ namespace { using reporter_base::get_error; }; + class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base { + public: + bloom_tree_reporter(nested_output &o) + : reporter_base(o) { + } + + void visit(bloom_tree_detail::missing_eras const &d) { + out() << "missing eras" << end_message(); + { + nested_output::nest _ = push(); + out() << d.get_desc() << end_message(); + out() << "Effected eras: [" << d.eras_.begin_.get() + << ", " << d.eras_.end_.get() << ")" << end_message(); + } + + mplus_error(FATAL); + } + + void visit(bloom_tree_detail::damaged_bloom_filter const &d) { + out() << "damaged bloom filter" << end_message(); + { + nested_output::nest _ = push(); + out() << d.get_desc() << end_message(); + out() << "Era: " << d.era_ << end_message(); + out() << "Missing bits: [" << d.missing_bits_.begin_.get() + << ", " << d.missing_bits_.end_.get() << ")" << end_message(); + } + + mplus_error(FATAL); + } + + using reporter_base::get_error; + }; + //-------------------------------- transaction_manager::ptr open_tm(block_manager<>::ptr bm) { @@ -145,16 +180,16 @@ namespace { return sb_rep.get_error(); -#if 0 superblock sb = read_superblock(bm); transaction_manager::ptr tm = open_tm(bm); - // FIXME: make an error class that's an instance of mplus + bloom_tree_reporter bt_rep(out); + era_detail_traits::ref_counter rc(tm); + bloom_tree bt(tm, rc); + check_bloom_tree(tm, bt, bt_rep); + return combine_errors(sb_rep.get_error(), - combine_errors(mapping_rep.get_error(), - combine_errors(hint_rep.get_error(), - discard_rep.get_error()))); -#endif + bt_rep.get_error()); } int check(string const &path, flags const &fs) { diff --git a/era/era_detail.h b/era/era_detail.h index 718afe4..7afaaaf 100644 --- a/era/era_detail.h +++ b/era/era_detail.h @@ -2,6 +2,7 @@ #define ERA_DETAIL_H #include "base/endian_utils.h" +#include "persistent-data/transaction_manager.h" //---------------------------------------------------------------- @@ -25,9 +26,17 @@ namespace era { uint64_t bloom_root; }; + struct era_detail_ref_counter { + era_detail_ref_counter(persistent_data::transaction_manager::ptr tm); + + void inc(persistent_data::block_address b); + void dec(persistent_data::block_address b); + }; + struct era_detail_traits { typedef era_detail_disk disk_type; typedef era_detail value_type; + typedef era_detail_ref_counter ref_counter; static void unpack(disk_type const &disk, value_type &value); static void pack(value_type const &value, disk_type &disk);