[era_check] Now checks bloom tree

Makefile.in

@@ -51,6 +51,7 @@ SOURCE=\
 	\
 	era/era_detail.cc \
 	era/superblock.cc \
+	era/bloom_tree.cc \
 	\
 	persistent-data/checksum.cc \
 	persistent-data/error_set.cc \
@@ -299,6 +300,7 @@ ERA_CHECK_SOURCE=\
 	base/error_state.cc \
 	base/endian_utils.cc \
 	\
+	era/bloom_tree.cc \
 	era/superblock.cc \
 	era/era_detail.cc \
 	\

era/bloom_tree.cc

@@ -0,0 +1,126 @@
+#include "era/bloom_tree.h"
+#include "persistent-data/data-structures/btree_damage_visitor.h"
+#include "persistent-data/data-structures/bitset.h"
+
+using namespace boost;
+using namespace era;
+using namespace bloom_tree_detail;
+using namespace persistent_data;
+using namespace std;
+
+//----------------------------------------------------------------
+
+missing_eras::missing_eras(string const &desc,
+			   run<uint32_t> const &eras)
+	: damage(desc),
+	  eras_(eras)
+{
+}
+
+void
+missing_eras::visit(damage_visitor &v) const {
+	v.visit(*this);
+}
+
+damaged_bloom_filter::damaged_bloom_filter(string const &desc,
+					   uint32_t era,
+					   run<uint32_t> missing_bits)
+	: damage(desc),
+	  era_(era),
+	  missing_bits_(missing_bits)
+{
+}
+
+void
+damaged_bloom_filter::visit(damage_visitor &v) const
+{
+	v.visit(*this);
+}
+
+//----------------------------------------------------------------
+
+namespace {
+	class ll_bloom_visitor : public bitset_detail::bitset_visitor {
+	public:
+		typedef persistent_data::transaction_manager::ptr tm_ptr;
+
+		ll_bloom_visitor(tm_ptr tm,
+				 bloom_tree_detail::bloom_visitor &bloom_v,
+				 bloom_tree_detail::damage_visitor &dv)
+			: tm_(tm),
+			  bloom_v_(bloom_v),
+			  dv_(dv) {
+		}
+
+		void visit(btree_path const &path, era_detail const &era) {
+			era_ = path[0];
+			bitset bs(tm_, era.bloom_root, era.nr_bits);
+			bs.walk_bitset(*this);
+		}
+
+		void visit(uint32_t index, bool value) {
+			bloom_v_.visit(index, value);
+		}
+
+		void visit(bitset_detail::missing_bits const &d) {
+			dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_));
+		}
+
+	private:
+		tm_ptr tm_;
+		uint64_t era_;
+		bloom_tree_detail::bloom_visitor &bloom_v_;
+		bloom_tree_detail::damage_visitor &dv_;
+	};
+
+	class ll_damage_visitor {
+	public:
+		ll_damage_visitor(damage_visitor &v)
+		: v_(v) {
+		}
+
+		virtual void visit(btree_path const &path,
+				   btree_detail::damage const &d) {
+			v_.visit(missing_eras(d.desc_, to_uint32(d.lost_keys_)));
+		}
+
+	private:
+		template <typename T>
+		run<uint32_t> to_uint32(run<T> const &r) {
+			return run<uint32_t>(optional<uint32_t>(r.begin_),
+					     optional<uint32_t>(r.end_));
+		}
+
+		damage_visitor &v_;
+	};
+}
+
+void
+era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
+		     bloom_tree const &tree,
+		     bloom_tree_detail::bloom_visitor &bloom_v,
+		     bloom_tree_detail::damage_visitor &dv)
+{
+	ll_bloom_visitor ll_bv(tm, bloom_v, dv);
+	ll_damage_visitor ll_dv(dv);
+	btree_visit_values(tree, ll_bv, ll_dv);
+}
+
+namespace {
+	class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor {
+	public:
+		void visit(uint32_t index, bool value) {
+		}
+	};
+};
+
+void
+era::check_bloom_tree(persistent_data::transaction_manager::ptr tm,
+		      bloom_tree const &tree,
+		      bloom_tree_detail::damage_visitor &dv)
+{
+	noop_bloom_visitor bv;
+	walk_bloom_tree(tm, tree, bv, dv);
+}
+
+//----------------------------------------------------------------

era/bloom_tree.h

@@ -1,112 +1,84 @@
 #ifndef ERA_BLOOM_TREE_H
 #define ERA_BLOOM_TREE_H
 
+#include "era/era_detail.h"
+#include "persistent-data/data-structures/btree.h"
+
 //----------------------------------------------------------------
 
 namespace era {
+	namespace bloom_tree_detail {
+		class damage_visitor;
+
+		class damage {
+		public:
+			damage(std::string const &desc)
+				: desc_(desc) {
+			}
+
+			virtual ~damage() {}
+			virtual void visit(damage_visitor &v) const = 0;
+
+			std::string const &get_desc() const {
+				return desc_;
+			}
+
+		private:
+			std::string desc_;
+		};
+
+		struct missing_eras : public damage {
+			missing_eras(std::string const &desc, run<uint32_t> const &eras);
+			virtual void visit(damage_visitor &v) const;
+
+			run<uint32_t> eras_;
+		};
+
+		struct damaged_bloom_filter : public damage {
+			damaged_bloom_filter(std::string const &desc,
+					     uint32_t era,
+					     run<uint32_t> missing_bits);
+			virtual void visit(damage_visitor &v) const;
+
+			uint32_t era_;
+			run<uint32_t> missing_bits_;
+		};
+
+		class damage_visitor {
+		public:
+			typedef boost::shared_ptr<damage_visitor> ptr;
+
+			virtual ~damage_visitor() {}
+
+			void visit(damage const &d) {
+				d.visit(*this);
+			}
+
+			virtual void visit(missing_eras const &d) = 0;
+			virtual void visit(damaged_bloom_filter const &d) = 0;
+		};
+
+		class bloom_visitor {
+		public:
+			typedef boost::shared_ptr<bloom_visitor> ptr;
+
+			virtual ~bloom_visitor() {}
+			virtual void visit(uint32_t index, bool value) = 0;
+		};
+	}
+
+	typedef persistent_data::btree<1, era_detail_traits> bloom_tree;
+
+	void walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
+			     bloom_tree const &tree,
+			     bloom_tree_detail::bloom_visitor &bloom_v,
+			     bloom_tree_detail::damage_visitor &dv);
+
+	void check_bloom_tree(persistent_data::transaction_manager::ptr tm,
+			      bloom_tree const &tree,
+			      bloom_tree_detail::damage_visitor &dv);
 }
 
 //----------------------------------------------------------------
 
 #endif
-
-
-#if 0
-	class dm_era {
-	public:
-		dm_era(block_address nr_blocks)
-		: nr_blocks_(nr_blocks),
-		  era_base_(0),
-		  base_(nr_blocks, false) {
-		}
-
-		set<block_address> blocks_written_since(unsigned era) const {
-
-		}
-
-		unsigned get_era() const {
-			return era_base_ + eras_.size() - 1;
-		}
-
-		void record_write(block_address b) {
-			current_era.record_write(b);
-		}
-
-		void resize(block_address new_size) {
-			nr_blocks_ = new_size;
-			push_era();
-			base_.resize(new_size, false);
-		}
-
-	private:
-		era_details &current_era() {
-			return eras_.back();
-		}
-
-		void need_new_era() {
-			// ???
-		}
-
-		void push_era() {
-			eras_.push_back(era(nr_blocks_));
-			if (eras_.size() > 100)
-				pop_era();
-		}
-
-		void pop_era() {
-			era_base_++;
-
-
-
-			eras_.pop_front();
-		}
-
-		static const unsigned NR_PROBES = 6;
-
-		class era_details {
-		public:
-			era_details(block_address nr_blocks)
-			: nr_blocks_(nr_blocks),
-			  f(power_bits(nr_blocks, NR_PROBES)) {
-			}
-
-			void record_write(block_address b) {
-				f.add(b);
-			}
-
-			void add_blocks_written(set<block_address &result) const {
-				for (block_address b = 0; b < nr_blocks; b++)
-					if (f.test(b))
-						result.insert(b);
-			}
-
-		private:
-			static unsigned power_bits(block_address nr_blocks) {
-				// We're expecting 1% of the cache to change per era
-				block_address expected_writes = nr_blocks / 100;
-
-				unsigned r = 1;
-				while ((1ull << r) < (16 * expected_writes))
-					r++;
-
-				return r;
-			}
-
-			typedef bloom_filter<block_address_bloom_traits> filter;
-
-			block_address nr_blocks;
-			filter f;
-		};
-
-		space_map::ptr setup_core_map() {
-			space_map::ptr sm(new core_map(NR_BLOCKS));
-			sm->inc(SUPERBLOCK);
-			return sm;
-		}
-
-		block_address nr_blocks_;
-		unsigned era_base_;
-		vector<bool> base_;
-		deque<era_details> eras_;
-	};
-#endif

era/era_check.cc

@@ -14,6 +14,7 @@
 
 #include "base/error_state.h"
 #include "base/nested_output.h"
+#include "era/bloom_tree.h"
 #include "era/superblock.h"
 #include "persistent-data/block.h"
 #include "persistent-data/file_utils.h"
@@ -90,6 +91,40 @@ namespace {
 		using reporter_base::get_error;
 	};
 
+	class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base {
+	public:
+		bloom_tree_reporter(nested_output &o)
+		: reporter_base(o) {
+		}
+
+		void visit(bloom_tree_detail::missing_eras const &d) {
+			out() << "missing eras" << end_message();
+			{
+				nested_output::nest _ = push();
+				out() << d.get_desc() << end_message();
+				out() << "Effected eras: [" << d.eras_.begin_.get()
+				      << ", " << d.eras_.end_.get() << ")" << end_message();
+			}
+
+			mplus_error(FATAL);
+		}
+
+		void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
+			out() << "damaged bloom filter" << end_message();
+			{
+				nested_output::nest _ = push();
+				out() << d.get_desc() << end_message();
+				out() << "Era: " << d.era_ << end_message();
+				out() << "Missing bits: [" << d.missing_bits_.begin_.get()
+				      << ", " << d.missing_bits_.end_.get() << ")" << end_message();
+			}
+
+			mplus_error(FATAL);
+		}
+
+		using reporter_base::get_error;
+	};
+
 	//--------------------------------
 
 	transaction_manager::ptr open_tm(block_manager<>::ptr bm) {
@@ -145,16 +180,16 @@ namespace {
 
 		return sb_rep.get_error();
 
-#if 0
 		superblock sb = read_superblock(bm);
 		transaction_manager::ptr tm = open_tm(bm);
 
-		// FIXME: make an error class that's an instance of mplus
+		bloom_tree_reporter bt_rep(out);
+		era_detail_traits::ref_counter rc(tm);
+		bloom_tree bt(tm, rc);
+		check_bloom_tree(tm, bt, bt_rep);
+
 		return combine_errors(sb_rep.get_error(),
-				      combine_errors(mapping_rep.get_error(),
+				      bt_rep.get_error());
-						     combine_errors(hint_rep.get_error(),
-								    discard_rep.get_error())));
-#endif
 	}
 
 	int check(string const &path, flags const &fs) {

era/era_detail.h

@@ -2,6 +2,7 @@
 #define ERA_DETAIL_H
 
 #include "base/endian_utils.h"
+#include "persistent-data/transaction_manager.h"
 
 //----------------------------------------------------------------
 
@@ -25,9 +26,17 @@ namespace era {
 		uint64_t bloom_root;
 	};
 
+	struct era_detail_ref_counter {
+		era_detail_ref_counter(persistent_data::transaction_manager::ptr tm);
+
+		void inc(persistent_data::block_address b);
+		void dec(persistent_data::block_address b);
+	};
+
 	struct era_detail_traits {
 		typedef era_detail_disk disk_type;
 		typedef era_detail value_type;
+		typedef era_detail_ref_counter ref_counter;
 
 		static void unpack(disk_type const &disk, value_type &value);
 		static void pack(value_type const &value, disk_type &disk);