persistent-data/data-structures/bloom_filter

Makefile.in

@@ -55,6 +55,7 @@ SOURCE=\
 	persistent-data/transaction_manager.cc \
 	\
 	persistent-data/data-structures/bitset.cc \
+	persistent-data/data-structures/bloom_filter.cc \
 	persistent-data/data-structures/btree.cc \
 	\
 	persistent-data/space_map.cc \

persistent-data/data-structures/bloom_filter.cc

@@ -0,0 +1,146 @@
+#include "persistent-data/data-structures/bloom_filter.h"
+
+#include <stdexcept>
+
+using namespace persistent_data;
+
+//----------------------------------------------------------------
+
+namespace {
+	static const uint64_t m1 = 0x9e37fffffffc0001UL;
+	static const unsigned bits = 18;
+
+	static uint32_t hash1(block_address const &b) {
+		return (b * m1) >> bits;
+	}
+
+	static uint32_t hash2(block_address const &b) {
+		uint32_t n = b;
+
+		n = n ^ (n >> 16);
+		n = n * 0x85ebca6bu;
+		n = n ^ (n >> 13);
+		n = n * 0xc2b2ae35u;
+		n = n ^ (n >> 16);
+
+		return n;
+	}
+
+	void check_power_of_two(unsigned nr_bits) {
+		if (nr_bits & (nr_bits - 1))
+			throw std::runtime_error("bloom filter needs a power of two nr_bits");
+	}
+}
+
+//----------------------------------------------------------------
+
+bloom_filter::bloom_filter(tm_ptr tm,
+			   unsigned nr_bits, unsigned nr_probes)
+	: tm_(tm),
+	  bits_(tm),
+	  nr_probes_(nr_probes),
+	  mask_(nr_bits - 1)
+{
+	check_power_of_two(nr_bits);
+	bits_.grow(nr_bits, false);
+}
+
+bloom_filter::bloom_filter(tm_ptr tm, block_address root,
+			   unsigned nr_bits, unsigned nr_probes)
+	: tm_(tm),
+	  bits_(tm, root, nr_bits),
+	  nr_probes_(nr_probes),
+	  mask_(nr_bits - 1)
+{
+	check_power_of_two(nr_bits);
+}
+
+block_address
+bloom_filter::get_root() const
+{
+	return bits_.get_root();
+}
+
+bool
+bloom_filter::test(uint64_t b)
+{
+	vector<unsigned> probes(nr_probes_);
+	fill_probes(b, probes);
+
+	for (unsigned p = 0; p < nr_probes_; p++)
+		if (!bits_.get(probes[p]))
+			return false;
+
+	return true;
+}
+
+void
+bloom_filter::set(uint64_t b)
+{
+	vector<unsigned> probes(nr_probes_);
+	fill_probes(b, probes);
+
+	for (unsigned p = 0; p < nr_probes_; p++)
+		bits_.set(probes[p], true);
+}
+
+void
+bloom_filter::flush()
+{
+	bits_.flush();
+}
+
+void
+bloom_filter::fill_probes(block_address b, vector<unsigned> &probes) const
+{
+	uint32_t h1 = hash1(b) & mask_;
+	uint32_t h2 = hash2(b) & mask_;
+
+	probes[0] = h1;
+	for (unsigned p = 1; p < nr_probes_; p++) {
+		h1 = (h1 + h2) & mask_;
+		h2 = (h2 + p) & mask_;
+		probes[p] = h1;
+	}
+}
+
+void
+bloom_filter::print_debug(ostream &out)
+{
+	print_residency(out);
+
+	map<unsigned, unsigned> runs;
+
+	for (unsigned i = 0; i < bits_.get_nr_bits();) {
+		bool v = bits_.get(i);
+		unsigned run_length = 1;
+
+		while (++i < bits_.get_nr_bits() && bits_.get(i) == v)
+			run_length++;
+
+		map<unsigned, unsigned>::iterator it = runs.find(run_length);
+		if (it != runs.end())
+			it->second++;
+		else
+			runs.insert(make_pair(run_length, 1));
+	}
+
+	{
+		map<unsigned, unsigned>::const_iterator it;
+		for (it = runs.begin(); it != runs.end(); ++it)
+			out << it->first << ": " << it->second << endl;
+	}
+}
+
+void
+bloom_filter::print_residency(ostream &out)
+{
+	unsigned count = 0;
+	for (unsigned i = 0; i < bits_.get_nr_bits(); i++)
+		if (bits_.get(i))
+			count++;
+
+	out << "residency: " << count << "/" << bits_.get_nr_bits() << endl;
+}
+
+//----------------------------------------------------------------

persistent-data/data-structures/bloom_filter.h

@@ -0,0 +1,47 @@
+#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BLOOM_FILTER_H
+#define PERSISTENT_DATA_DATA_STRUCTURES_BLOOM_FILTER_H
+
+#include "persistent-data/transaction_manager.h"
+#include "persistent-data/data-structures/bitset.h"
+
+#include <boost/shared_ptr.hpp>
+
+//----------------------------------------------------------------
+
+namespace persistent_data {
+	class bloom_filter {
+	public:
+		typedef boost::shared_ptr<bloom_filter> ptr;
+		typedef typename persistent_data::transaction_manager::ptr tm_ptr;
+
+		// nr_bits must be a power of two
+		bloom_filter(tm_ptr tm,
+			     unsigned nr_bits, unsigned nr_probes);
+
+		bloom_filter(tm_ptr tm, block_address root,
+			     unsigned nr_bits_power, unsigned nr_probes);
+
+		block_address get_root() const;
+
+		bool test(uint64_t b); // not const due to caching effects in bitset
+		void set(uint64_t b);
+		void flush();
+
+		void print_debug(ostream &out);
+
+	private:
+		void print_residency(ostream &out);
+
+		void fill_probes(block_address b, vector<unsigned> &probes) const;
+
+		tm_ptr tm_;
+		unsigned nr_bits_;
+		persistent_data::bitset bits_;
+		unsigned nr_probes_;
+		uint64_t mask_;
+	};
+}
+
+//----------------------------------------------------------------
+
+#endif

unit-tests/bloom_filter_t.cc

@@ -1,4 +1,5 @@
 #include "gmock/gmock.h"
+#include "persistent-data/data-structures/bloom_filter.h"
 #include "persistent-data/transaction_manager.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/data-structures/array_block.h"
@@ -19,221 +20,19 @@ using namespace testing;
 //----------------------------------------------------------------
 
 namespace {
-	struct block_address_bloom_traits {
+	block_address const BLOCK_SIZE = 4096;
-		typedef block_address value_type;
+	block_address const NR_BLOCKS = 102400;
-
+	block_address const SUPERBLOCK = 0;
-		static const uint64_t ones = ~0ull;
-		static const uint64_t m1 = 0x9e37fffffffc0001UL;
-		static const uint64_t m2 = ones - 82;
-
-		static const unsigned bits = 18;
-
-		static uint64_t hash1(block_address const &b) {
-			return (b * m1) >> bits;
-		}
-
-		static uint64_t hash2(block_address const &b) {
-			uint32_t n = b;
-
-			n = n ^ (n >> 16);
-			n = n * 0x85ebca6bu;
-			n = n ^ (n >> 13);
-			n = n * 0xc2b2ae35u;
-			n = n ^ (n >> 16);
-
-			return n;
-		}
-
-		static uint64_t hash3(block_address const &b) {
-			return (b * m2) >> bits;
-		}
-	};
-
-	template <class Traits>
-	class bloom_filter {
-	public:
-		bloom_filter(unsigned nr_bits_power, unsigned nr_probes)
-			: bits_(1ull << nr_bits_power, false),
-			  nr_probes_(nr_probes),
-			  mask_((1ull << nr_bits_power) - 1) {
-
-			cerr << "nr entries = " << bits_.size() << ", mask = " << mask_ << endl;
-		}
-
-		bool test(typename Traits::value_type const &v) {
-			vector<uint32_t> probes(nr_probes_);
-			fill_probes(v, probes);
-
-			for (unsigned p = 0; p < nr_probes_; p++)
-				if (!bits_.at(probes[p]))
-					return false;
-
-			return true;
-		}
-
-		void add(typename Traits::value_type const &v) {
-			vector<uint32_t> probes(nr_probes_);
-			fill_probes(v, probes);
-
-			for (unsigned p = 0; p < nr_probes_; p++) {
-				//cerr << probes[p] << ", ";
-				bits_.at(probes[p]) = true;
-			}
-			//cerr << endl;
-		}
-
-		void dump() const {
-			residency();
-
-			map<unsigned, unsigned> runs;
-
-			for (unsigned i = 0; i < bits_.size();) {
-				bool v = bits_[i];
-				unsigned run_length = 1;
-
-				while (bits_[++i] == v && i < bits_.size())
-					run_length++;
-
-				map<unsigned, unsigned>::iterator it = runs.find(run_length);
-				if (it != runs.end())
-					it->second++;
-				else
-					runs.insert(make_pair(run_length, 1));
-			}
-
-			{
-				map<unsigned, unsigned>::const_iterator it;
-				for (it = runs.begin(); it != runs.end(); ++it)
-					cout << it->first << ": " << it->second << endl;
-			}
-		}
-
-		void residency() const {
-			unsigned count = 0;
-			for (unsigned i = 0; i < bits_.size(); i++)
-				if (bits_[i])
-					count++;
-
-			cout << "residency: " << count << "/" << bits_.size() << endl;
-		}
-
-	private:
-		void fill_probes(typename Traits::value_type const &v, vector<uint32_t> &probes) {
-			uint32_t h1 = Traits::hash1(v) & mask_;
-			uint32_t h2 = Traits::hash2(v) & mask_;
-
-			probes[0] = h1;
-			for (unsigned p = 1; p < nr_probes_; p++) {
-				h1 = (h1 + h2) & mask_;
-				h2 = (h2 + p) & mask_;
-				probes[p] = h1;
-			}
-		}
-
-		vector<bool> bits_;
-		unsigned nr_probes_;
-		uint64_t mask_;
-	};
-
-	//--------------------------------
-#if 0
-	class dm_era {
-	public:
-		dm_era(block_address nr_blocks)
-		: nr_blocks_(nr_blocks),
-		  era_base_(0),
-		  base_(nr_blocks, false) {
-		}
-
-		set<block_address> blocks_written_since(unsigned era) const {
-
-		}
-
-		unsigned get_era() const {
-			return era_base_ + eras_.size() - 1;
-		}
-
-		void record_write(block_address b) {
-			current_era.record_write(b);
-		}
-
-		void resize(block_address new_size) {
-			nr_blocks_ = new_size;
-			push_era();
-			base_.resize(new_size, false);
-		}
-
-	private:
-		era_details &current_era() {
-			return eras_.back();
-		}
-
-		void need_new_era() {
-			// ???
-		}
-
-		void push_era() {
-			eras_.push_back(era(nr_blocks_));
-			if (eras_.size() > 100)
-				pop_era();
-		}
-
-		void pop_era() {
-			era_base_++;
-
-
-
-			eras_.pop_front();
-		}
-
-		static const unsigned NR_PROBES = 6;
-
-		class era_details {
-		public:
-			era_details(block_address nr_blocks)
-			: nr_blocks_(nr_blocks),
-			  f(power_bits(nr_blocks, NR_PROBES)) {
-			}
-
-			void record_write(block_address b) {
-				f.add(b);
-			}
-
-			void add_blocks_written(set<block_address &result) const {
-				for (block_address b = 0; b < nr_blocks; b++)
-					if (f.test(b))
-						result.insert(b);
-			}
-
-		private:
-			static unsigned power_bits(block_address nr_blocks) {
-				// We're expecting 1% of the cache to change per era
-				block_address expected_writes = nr_blocks / 100;
-
-				unsigned r = 1;
-				while ((1ull << r) < (16 * expected_writes))
-					r++;
-
-				return r;
-			}
-
-			typedef bloom_filter<block_address_bloom_traits> filter;
-
-			block_address nr_blocks;
-			filter f;
-		};
-
-		block_address nr_blocks_;
-		unsigned era_base_;
-		vector<bool> base_;
-		deque<era_details> eras_;
-	};
-#endif
 
 	//--------------------------------
 
 	class BloomFilterTests : public Test {
 	public:
+		BloomFilterTests()
+			: bm_(create_bm<BLOCK_SIZE>(NR_BLOCKS)),
+			  sm_(setup_core_map()),
+			  tm_(new transaction_manager(bm_, sm_)) {
+		}
 
 		set<block_address> generate_random_blocks(unsigned count,
 							  block_address max = std::numeric_limits<uint64_t>::max()) {
@@ -251,58 +50,104 @@ namespace {
 
 			return r;
 		}
+
+		void commit() {
+			block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK));
+		}
+
+		space_map::ptr setup_core_map() {
+			space_map::ptr sm(new core_map(NR_BLOCKS));
+			sm->inc(SUPERBLOCK);
+			return sm;
+		}
+
+		with_temp_directory dir_;
+		block_manager<>::ptr bm_;
+		space_map::ptr sm_;
+		transaction_manager::ptr tm_;
 	};
 }
 
 //----------------------------------------------------------------
 
+TEST_F(BloomFilterTests, nr_bits_must_be_a_power_of_two)
+{
+	ASSERT_THROW(bloom_filter f(tm_, 1023, 3), runtime_error);
+}
+
 TEST_F(BloomFilterTests, can_create_a_bloom_filter)
 {
-	bloom_filter<block_address_bloom_traits> f(10, 3);
+	bloom_filter f(tm_, 1024, 3);
 }
 
 TEST_F(BloomFilterTests, no_false_negatives)
 {
-	bloom_filter<block_address_bloom_traits> f(12, 6);
+	bloom_filter f(tm_, 4096, 6);
 	set<block_address> bs = generate_random_blocks(1000);
 
 	set<block_address>::const_iterator it;
 	for (it = bs.begin(); it != bs.end(); ++it)
-		f.add(*it);
+		f.set(*it);
 
 	for (it = bs.begin(); it != bs.end(); ++it)
 		ASSERT_THAT(f.test(*it), Eq(true));
+}
 
-	f.dump();
+TEST_F(BloomFilterTests, reload_works)
+{
+	block_address root;
+	set<block_address> bs = generate_random_blocks(1000);
+
+	{
+		bloom_filter f(tm_, 4096, 6);
+
+		set<block_address>::const_iterator it;
+		for (it = bs.begin(); it != bs.end(); ++it)
+			f.set(*it);
+
+		f.flush();
+		root = f.get_root();
+		commit();
+	}
+
+	{
+		bloom_filter f(tm_, root, 4096, 6);
+
+		set<block_address>::const_iterator it;
+		for (it = bs.begin(); it != bs.end(); ++it)
+			ASSERT_THAT(f.test(*it), Eq(true));
+	}
 }
 
 TEST_F(BloomFilterTests, count_false_positives)
 {
-	block_address nr_blocks = 128 * 1024 * 1024;
+	block_address nr_blocks = 1024 * 1024;
 	block_address written_blocks = nr_blocks / 100;
 
 	unsigned shift = 1;
 
 	while ((1ull << shift) < (16 * written_blocks))
 		shift++;
-	cerr << "bitset " << ((1 << shift) / (8 * 1024)) << "k" << endl;
+	cerr << "bitset size: " << ((1 << shift) / (8 * 1024)) << "k" << endl;
+
+	bloom_filter f(tm_, 1 << shift, 6);
 
-	bloom_filter<block_address_bloom_traits> f(shift, 6);
 	set<block_address> bs = generate_random_blocks(written_blocks, nr_blocks);
 	set<block_address>::const_iterator it;
 
 	for (it = bs.begin(); it != bs.end(); ++it)
-		f.add(*it);
+		f.set(*it);
 
-	f.dump();
+	// f.print_debug(cerr);
 
 	unsigned count = 0;
 	for (unsigned i = 0; i < nr_blocks; i++)
 		if (!bs.count(i) && f.test(i))
 			count++;
 
-	cerr << count << "false positives out of " << nr_blocks << endl;
+	cerr << count << " false positives out of " << nr_blocks << ", "
-	cerr << static_cast<double>(count * 100) / static_cast<double>(nr_blocks) << "%" << endl;
+	     << static_cast<double>(count * 100) / static_cast<double>(nr_blocks)
+	     << "%" << endl;
 }
 
 //----------------------------------------------------------------