From 8531a2befa45e1d71332be055f0a5d155bac58d9 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 29 Jan 2014 21:37:25 +0000 Subject: [PATCH] [era] switch from bloom filters to writesets --- Makefile.in | 39 ++++- era/emitter.h | 6 +- era/era_check.cc | 24 +-- era/era_detail.cc | 23 +-- era/era_detail.h | 13 +- era/era_dump.cc | 2 +- era/era_invalidate.cc | 114 +++++++++++++++ era/metadata.cc | 6 +- era/metadata.h | 4 +- era/metadata_dump.cc | 40 ++--- era/superblock.cc | 16 +- era/superblock.h | 2 +- era/{bloom_tree.cc => writeset_tree.cc} | 63 ++++---- era/{bloom_tree.h => writeset_tree.h} | 43 +++--- era/xml_format.cc | 14 +- persistent-data/block.tcc | 2 +- unit-tests/array_t.cc | 1 + unit-tests/bloom_filter_t.cc | 185 ++++++++++++++++++++++-- unit-tests/btree_counter_t.cc | 1 + unit-tests/btree_t.cc | 1 + 20 files changed, 439 insertions(+), 160 deletions(-) create mode 100644 era/era_invalidate.cc rename era/{bloom_tree.cc => writeset_tree.cc} (54%) rename era/{bloom_tree.h => writeset_tree.h} (54%) diff --git a/Makefile.in b/Makefile.in index f199a80..6e88395 100644 --- a/Makefile.in +++ b/Makefile.in @@ -27,6 +27,7 @@ PROGRAMS=\ \ era_check \ era_dump \ + era_invalidate \ \ thin_check \ thin_dump \ @@ -53,7 +54,7 @@ SOURCE=\ era/era_array.cc \ era/era_detail.cc \ era/superblock.cc \ - era/bloom_tree.cc \ + era/writeset_tree.cc \ era/metadata.cc \ era/metadata_dump.cc \ era/xml_format.cc \ @@ -305,7 +306,7 @@ ERA_CHECK_SOURCE=\ base/error_state.cc \ base/endian_utils.cc \ \ - era/bloom_tree.cc \ + era/writeset_tree.cc \ era/era_detail.cc \ era/era_array.cc \ era/metadata.cc \ @@ -335,7 +336,7 @@ ERA_DUMP_SOURCE=\ base/error_state.cc \ base/endian_utils.cc \ \ - era/bloom_tree.cc \ + era/writeset_tree.cc \ era/era_detail.cc \ era/era_array.cc \ era/metadata.cc \ @@ -362,6 +363,38 @@ era_dump: $(ERA_DUMP_OBJECTS) era/era_dump.o @echo " [LD] $@" $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) +ERA_INVALIDATE_SOURCE=\ + base/base64.cc \ + base/error_state.cc \ + base/endian_utils.cc \ + \ + era/writeset_tree.cc \ + era/era_detail.cc \ + era/era_array.cc \ + era/metadata.cc \ + era/metadata_dump.cc \ + era/superblock.cc \ + era/xml_format.cc \ + \ + persistent-data/checksum.cc \ + persistent-data/error_set.cc \ + persistent-data/file_utils.cc \ + persistent-data/hex_dump.cc \ + persistent-data/lock_tracker.cc \ + persistent-data/data-structures/btree.cc \ + persistent-data/data-structures/bitset.cc \ + persistent-data/space_map.cc \ + persistent-data/space-maps/disk.cc \ + persistent-data/space-maps/recursive.cc \ + persistent-data/space-maps/careful_alloc.cc \ + persistent-data/transaction_manager.cc \ + +ERA_INVALIDATE_OBJECTS=$(subst .cc,.o,$(ERA_INVALIDATE_SOURCE)) + +era_invalidate: $(ERA_INVALIDATE_OBJECTS) era/era_invalidate.o + @echo " [LD] $@" + $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) + #---------------------------------------------------------------- DEPEND_FILES=\ diff --git a/era/emitter.h b/era/emitter.h index 0fb7520..c175384 100644 --- a/era/emitter.h +++ b/era/emitter.h @@ -20,9 +20,9 @@ namespace era { uint32_t current_era) = 0; virtual void end_superblock() = 0; - virtual void begin_bloom(uint32_t era, uint32_t nr_bits, pd::block_address nr_blocks) = 0; - virtual void bloom_bit(uint32_t bit, bool value) = 0; - virtual void end_bloom() = 0; + virtual void begin_writeset(uint32_t era, uint32_t nr_bits) = 0; + virtual void writeset_bit(uint32_t bit, bool value) = 0; + virtual void end_writeset() = 0; virtual void begin_era_array() = 0; virtual void era(pd::block_address block, uint32_t era) = 0; diff --git a/era/era_check.cc b/era/era_check.cc index de65106..3e3f1fd 100644 --- a/era/era_check.cc +++ b/era/era_check.cc @@ -14,7 +14,7 @@ #include "base/error_state.h" #include "base/nested_output.h" -#include "era/bloom_tree.h" +#include "era/writeset_tree.h" #include "era/era_array.h" #include "era/superblock.h" #include "persistent-data/block.h" @@ -92,14 +92,14 @@ namespace { using reporter_base::get_error; }; - class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base { + class writeset_tree_reporter : public writeset_tree_detail::damage_visitor, reporter_base { public: - bloom_tree_reporter(nested_output &o) + writeset_tree_reporter(nested_output &o) : reporter_base(o) { } - void visit(bloom_tree_detail::missing_eras const &d) { - out() << "missing eras from bloom tree" << end_message(); + void visit(writeset_tree_detail::missing_eras const &d) { + out() << "missing eras from writeset tree" << end_message(); { nested_output::nest _ = push(); out() << d.get_desc() << end_message(); @@ -110,8 +110,8 @@ namespace { mplus_error(FATAL); } - void visit(bloom_tree_detail::damaged_bloom_filter const &d) { - out() << "damaged bloom filter" << end_message(); + void visit(writeset_tree_detail::damaged_writeset const &d) { + out() << "damaged writeset" << end_message(); { nested_output::nest _ = push(); out() << d.get_desc() << end_message(); @@ -216,22 +216,22 @@ namespace { superblock sb = read_superblock(bm); transaction_manager::ptr tm = open_tm(bm); - bloom_tree_reporter bt_rep(out); + writeset_tree_reporter wt_rep(out); { era_detail_traits::ref_counter rc(tm); - bloom_tree bt(tm, sb.bloom_tree_root, rc); - check_bloom_tree(tm, bt, bt_rep); + writeset_tree wt(tm, sb.writeset_tree_root, rc); + check_writeset_tree(tm, wt, wt_rep); } era_array_reporter ea_rep(out); { uint32_traits::ref_counter rc; era_array ea(tm, rc, sb.era_array_root, sb.nr_blocks); - check_era_array(ea, ea_rep); + check_era_array(ea, sb.current_era, ea_rep); } return combine_errors(sb_rep.get_error(), - combine_errors(bt_rep.get_error(), + combine_errors(wt_rep.get_error(), ea_rep.get_error())); } diff --git a/era/era_detail.cc b/era/era_detail.cc index 7a88fcc..587102d 100644 --- a/era/era_detail.cc +++ b/era/era_detail.cc @@ -7,37 +7,18 @@ using namespace era; //---------------------------------------------------------------- -namespace { -#if 0 - le32 pack_hash_detail(uint32_t hash1, uint32_t hash2, uint32_t nr_probes) { - throw std::runtime_error("not implemented"); - } - - void unpack_hash_detail(le32 packed, uint32_t &hash1, uint32_t &hash2, uint32_t &nr_probes) { - throw std::runtime_error("not implemented"); - } -#endif -} - void era_detail_traits::unpack(disk_type const &disk, value_type &value) { - value.nr_blocks = to_cpu(disk.nr_blocks); value.nr_bits = to_cpu(disk.nr_bits); - value.nr_set = to_cpu(disk.nr_set); - value.bloom_root = to_cpu(disk.bloom_root); - - //unpack_hash_detail(disk.hash_fns_and_probes, value.hash1, value.hash2, value.nr_probes); + value.writeset_root = to_cpu(disk.writeset_root); } void era_detail_traits::pack(value_type const &value, disk_type &disk) { - disk.nr_blocks = to_disk(value.nr_blocks); disk.nr_bits = to_disk(value.nr_bits); - disk.nr_set = to_disk(value.nr_set); - disk.bloom_root = to_disk(value.bloom_root); -// disk.hash_fns_and_probes = pack_hash_detail(value.hash1, value.hash2, value.nr_probes); + disk.writeset_root = to_disk(value.writeset_root); } //---------------------------------------------------------------- diff --git a/era/era_detail.h b/era/era_detail.h index 6cacc09..8cb8b06 100644 --- a/era/era_detail.h +++ b/era/era_detail.h @@ -8,22 +8,13 @@ namespace era { struct era_detail_disk { - base::le32 nr_blocks; base::le32 nr_bits; - base::le32 nr_set; - base::le64 bloom_root; + base::le64 writeset_root; } __attribute__ ((packed)); struct era_detail { - uint32_t nr_blocks; uint32_t nr_bits; - uint32_t nr_set; - - uint32_t hash1; - uint32_t hash2; - uint32_t nr_probes; - - uint64_t bloom_root; + uint64_t writeset_root; }; // FIXME: implement diff --git a/era/era_dump.cc b/era/era_dump.cc index 1f13abd..a2c82aa 100644 --- a/era/era_dump.cc +++ b/era/era_dump.cc @@ -5,7 +5,7 @@ #include "version.h" #include "era/era_array.h" -#include "era/bloom_tree.h" +#include "era/writeset_tree.h" #include "era/metadata.h" #include "era/metadata_dump.h" #include "era/xml_format.h" diff --git a/era/era_invalidate.cc b/era/era_invalidate.cc new file mode 100644 index 0000000..c96b286 --- /dev/null +++ b/era/era_invalidate.cc @@ -0,0 +1,114 @@ +#include +#include +#include +#include + +#include "version.h" +#include "era/era_array.h" +#include "era/writeset_tree.h" +#include "era/metadata.h" +#include "era/xml_format.h" +#include "persistent-data/file_utils.h" + +using namespace era; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + struct flags { + flags() { + } + }; + + //-------------------------------- + + void emit_blocks() { + + } + + //-------------------------------- + + string const STDOUT_PATH("-"); + + bool want_stdout(string const &output) { + return output == STDOUT_PATH; + } + + int invalidate(string const &dev, string const &output, flags const &fs) { + try { + block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY); + metadata::ptr md(new metadata(bm, metadata::OPEN)); + + if (want_stdout(output)) { + emitter::ptr e = create_xml_emitter(cout); + //emit_blocks(md, e, fs); + } else { + ofstream out(output.c_str()); + emitter::ptr e = create_xml_emitter(out); + //emit_blocks(md, e, fs); + } + + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + + return 0; + } + + void usage(ostream &out, string const &cmd) { + out << "Usage: " << cmd << " [options] {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-o }" << endl + << " {-V|--version}" << endl; + } +} + +//---------------------------------------------------------------- + +int main(int argc, char **argv) +{ + int c; + flags fs; + string output("-"); + char const shortopts[] = "ho:V"; + + option const longopts[] = { + { "help", no_argument, NULL, 'h' }, + { "output", required_argument, NULL, 'o' }, + { "version", no_argument, NULL, 'V' }, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout, basename(argv[0])); + return 0; + + case 'o': + output = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr, basename(argv[0])); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr, basename(argv[0])); + return 1; + } + + return invalidate(argv[optind], output, fs); +} + +//---------------------------------------------------------------- diff --git a/era/metadata.cc b/era/metadata.cc index 270bd15..ab2eb7b 100644 --- a/era/metadata.cc +++ b/era/metadata.cc @@ -47,9 +47,9 @@ metadata::open_metadata(block_manager<>::ptr bm) tm_ = open_tm(bm); sb_ = read_superblock(tm_->get_bm()); - bloom_tree_ = bloom_tree::ptr(new bloom_tree(tm_, - sb_.bloom_tree_root, - era_detail_traits::ref_counter(tm_))); + writeset_tree_ = writeset_tree::ptr(new writeset_tree(tm_, + sb_.writeset_tree_root, + era_detail_traits::ref_counter(tm_))); era_array_ = era_array::ptr(new era_array(tm_, uint32_traits::ref_counter(), diff --git a/era/metadata.h b/era/metadata.h index 0d8a23b..ca31a50 100644 --- a/era/metadata.h +++ b/era/metadata.h @@ -10,7 +10,7 @@ #include "persistent-data/transaction_manager.h" #include "era/superblock.h" -#include "era/bloom_tree.h" +#include "era/writeset_tree.h" #include "era/era_array.h" //---------------------------------------------------------------- @@ -34,7 +34,7 @@ namespace era { tm::ptr tm_; superblock sb_; checked_space_map::ptr metadata_sm_; - bloom_tree::ptr bloom_tree_; + writeset_tree::ptr writeset_tree_; era_array::ptr era_array_; private: diff --git a/era/metadata_dump.cc b/era/metadata_dump.cc index edf4a1d..e7eac5f 100644 --- a/era/metadata_dump.cc +++ b/era/metadata_dump.cc @@ -17,42 +17,42 @@ namespace { "perhaps you wanted to run with --repair ?"); } - class bloom_tree_emitter : public bloom_tree_detail::bloom_visitor { + class writeset_tree_emitter : public writeset_tree_detail::writeset_visitor { public: - bloom_tree_emitter(emitter::ptr e) + writeset_tree_emitter(emitter::ptr e) : e_(e) { } - virtual void bloom_begin(uint32_t era, uint32_t nr_blocks, uint32_t nr_bits, uint32_t nr_set) { - e_->begin_bloom(era, nr_bits, nr_blocks); + virtual void writeset_begin(uint32_t era, uint32_t nr_bits) { + e_->begin_writeset(era, nr_bits); } virtual void bit(uint32_t bit, bool value) { - e_->bloom_bit(bit, value); + e_->writeset_bit(bit, value); } - virtual void bloom_end() { - e_->end_bloom(); + virtual void writeset_end() { + e_->end_writeset(); } private: emitter::ptr e_; }; - struct ignore_bloom_tree_damage : public bloom_tree_detail::damage_visitor { - void visit(bloom_tree_detail::missing_eras const &d) { + struct ignore_writeset_tree_damage : public writeset_tree_detail::damage_visitor { + void visit(writeset_tree_detail::missing_eras const &d) { } - void visit(bloom_tree_detail::damaged_bloom_filter const &d) { + void visit(writeset_tree_detail::damaged_writeset const &d) { } }; - struct fatal_bloom_tree_damage : public bloom_tree_detail::damage_visitor { - void visit(bloom_tree_detail::missing_eras const &d) { + struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor { + void visit(writeset_tree_detail::missing_eras const &d) { raise_metadata_damage(); } - void visit(bloom_tree_detail::damaged_bloom_filter const &d) { + void visit(writeset_tree_detail::damaged_writeset const &d) { raise_metadata_damage(); } }; @@ -104,15 +104,15 @@ era::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) sb.current_era); { { - bloom_tree_emitter visitor(e); + writeset_tree_emitter visitor(e); - ignore_bloom_tree_damage ignore; - fatal_bloom_tree_damage fatal; - bloom_tree_detail::damage_visitor &dv = repair ? - static_cast(ignore) : - static_cast(fatal); + ignore_writeset_tree_damage ignore; + fatal_writeset_tree_damage fatal; + writeset_tree_detail::damage_visitor &dv = repair ? + static_cast(ignore) : + static_cast(fatal); - walk_bloom_tree(md->tm_, *md->bloom_tree_, visitor, dv); + walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv); } e->begin_era_array(); diff --git a/era/superblock.cc b/era/superblock.cc index 99e51e0..0572314 100644 --- a/era/superblock.cc +++ b/era/superblock.cc @@ -34,7 +34,7 @@ namespace { le32 current_era; era_detail_disk current_detail; - le64 bloom_tree_root; + le64 writeset_tree_root; le64 era_array_root; } __attribute__ ((packed)); @@ -116,7 +116,7 @@ superblock::superblock() metadata_block_size(8), nr_blocks(0), current_era(0), - bloom_tree_root(0), + writeset_tree_root(0), era_array_root(0) { memset(uuid, 0, sizeof(uuid)); @@ -141,7 +141,7 @@ superblock_traits::unpack(disk_type const &disk, value_type &value) value.nr_blocks = to_cpu(disk.nr_blocks); value.current_era = to_cpu(disk.current_era); era_detail_traits::unpack(disk.current_detail, value.current_detail); - value.bloom_tree_root = to_cpu(disk.bloom_tree_root); + value.writeset_tree_root = to_cpu(disk.writeset_tree_root); value.era_array_root = to_cpu(disk.era_array_root); } @@ -161,7 +161,7 @@ superblock_traits::pack(value_type const &value, disk_type &disk) disk.nr_blocks = to_disk(value.nr_blocks); disk.current_era = to_disk(value.current_era); era_detail_traits::pack(value.current_detail, disk.current_detail); - disk.bloom_tree_root = to_disk(value.bloom_tree_root); + disk.writeset_tree_root = to_disk(value.writeset_tree_root); disk.era_array_root = to_disk(value.era_array_root); } @@ -282,8 +282,8 @@ era::check_superblock(superblock const &sb, visitor.visit(superblock_invalid(msg.str())); } - if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) { - string msg("bloom tree root points back to the superblock"); + if (sb.writeset_tree_root == SUPERBLOCK_LOCATION) { + string msg("writeset tree root points back to the superblock"); visitor.visit(superblock_invalid(msg)); } @@ -292,9 +292,9 @@ era::check_superblock(superblock const &sb, visitor.visit(superblock_invalid(msg)); } - if (sb.bloom_tree_root == sb.era_array_root) { + if (sb.writeset_tree_root == sb.era_array_root) { ostringstream msg; - msg << "bloom tree root and era array both point to the same block: " + msg << "writeset tree root and era array both point to the same block: " << sb.era_array_root; visitor.visit(superblock_invalid(msg.str())); } diff --git a/era/superblock.h b/era/superblock.h index 2ff1179..346582e 100644 --- a/era/superblock.h +++ b/era/superblock.h @@ -59,7 +59,7 @@ namespace era { era_detail current_detail; // A btree of undigested era_details - uint64_t bloom_tree_root; + uint64_t writeset_tree_root; // Big array holding the digested era/block info. uint64_t era_array_root; diff --git a/era/bloom_tree.cc b/era/writeset_tree.cc similarity index 54% rename from era/bloom_tree.cc rename to era/writeset_tree.cc index b49e533..763427e 100644 --- a/era/bloom_tree.cc +++ b/era/writeset_tree.cc @@ -1,10 +1,10 @@ -#include "era/bloom_tree.h" +#include "era/writeset_tree.h" #include "persistent-data/data-structures/btree_damage_visitor.h" #include "persistent-data/data-structures/bitset.h" using namespace boost; using namespace era; -using namespace bloom_tree_detail; +using namespace writeset_tree_detail; using namespace persistent_data; using namespace std; @@ -22,9 +22,9 @@ missing_eras::visit(damage_visitor &v) const { v.visit(*this); } -damaged_bloom_filter::damaged_bloom_filter(string const &desc, - uint32_t era, - run missing_bits) +damaged_writeset::damaged_writeset(string const &desc, + uint32_t era, + run missing_bits) : damage(desc), era_(era), missing_bits_(missing_bits) @@ -32,7 +32,7 @@ damaged_bloom_filter::damaged_bloom_filter(string const &desc, } void -damaged_bloom_filter::visit(damage_visitor &v) const +damaged_writeset::visit(damage_visitor &v) const { v.visit(*this); } @@ -40,39 +40,39 @@ damaged_bloom_filter::visit(damage_visitor &v) const //---------------------------------------------------------------- namespace { - class ll_bloom_visitor : public bitset_detail::bitset_visitor { + class ll_writeset_visitor : public bitset_detail::bitset_visitor { public: typedef persistent_data::transaction_manager::ptr tm_ptr; - ll_bloom_visitor(tm_ptr tm, - bloom_tree_detail::bloom_visitor &bloom_v, - bloom_tree_detail::damage_visitor &dv) + ll_writeset_visitor(tm_ptr tm, + writeset_tree_detail::writeset_visitor &writeset_v, + writeset_tree_detail::damage_visitor &dv) : tm_(tm), - bloom_v_(bloom_v), + writeset_v_(writeset_v), dv_(dv) { } void visit(btree_path const &path, era_detail const &era) { era_ = path[0]; - bitset bs(tm_, era.bloom_root, era.nr_bits); - bloom_v_.bloom_begin(era_, era.nr_blocks, era.nr_bits, era.nr_set); + bitset bs(tm_, era.writeset_root, era.nr_bits); + writeset_v_.writeset_begin(era_, era.nr_bits); bs.walk_bitset(*this); - bloom_v_.bloom_end(); + writeset_v_.writeset_end(); } void visit(uint32_t index, bool value) { - bloom_v_.bit(index, value); + writeset_v_.bit(index, value); } void visit(bitset_detail::missing_bits const &d) { - dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_)); + dv_.visit(writeset_tree_detail::damaged_writeset("missing bits", era_, d.keys_)); } private: tm_ptr tm_; uint64_t era_; - bloom_tree_detail::bloom_visitor &bloom_v_; - bloom_tree_detail::damage_visitor &dv_; + writeset_tree_detail::writeset_visitor &writeset_v_; + writeset_tree_detail::damage_visitor &dv_; }; class ll_damage_visitor { @@ -98,38 +98,37 @@ namespace { } void -era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm, - bloom_tree const &tree, - bloom_tree_detail::bloom_visitor &bloom_v, - bloom_tree_detail::damage_visitor &dv) +era::walk_writeset_tree(persistent_data::transaction_manager::ptr tm, + writeset_tree const &tree, + writeset_tree_detail::writeset_visitor &writeset_v, + writeset_tree_detail::damage_visitor &dv) { - ll_bloom_visitor ll_bv(tm, bloom_v, dv); + ll_writeset_visitor ll_bv(tm, writeset_v, dv); ll_damage_visitor ll_dv(dv); btree_visit_values(tree, ll_bv, ll_dv); } namespace { - class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor { + class noop_writeset_visitor : public writeset_tree_detail::writeset_visitor { public: - void bloom_begin(uint32_t era, uint32_t nr_blocks, - uint32_t nr_bits, uint32_t nr_set) { + void writeset_begin(uint32_t era, uint32_t nr_bits) { } void bit(uint32_t index, bool value) { } - void bloom_end() { + void writeset_end() { } }; }; void -era::check_bloom_tree(persistent_data::transaction_manager::ptr tm, - bloom_tree const &tree, - bloom_tree_detail::damage_visitor &dv) +era::check_writeset_tree(persistent_data::transaction_manager::ptr tm, + writeset_tree const &tree, + writeset_tree_detail::damage_visitor &dv) { - noop_bloom_visitor bv; - walk_bloom_tree(tm, tree, bv, dv); + noop_writeset_visitor bv; + walk_writeset_tree(tm, tree, bv, dv); } //---------------------------------------------------------------- diff --git a/era/bloom_tree.h b/era/writeset_tree.h similarity index 54% rename from era/bloom_tree.h rename to era/writeset_tree.h index ad4bbd6..9157ae6 100644 --- a/era/bloom_tree.h +++ b/era/writeset_tree.h @@ -1,5 +1,5 @@ -#ifndef ERA_BLOOM_TREE_H -#define ERA_BLOOM_TREE_H +#ifndef ERA_WRITESET_TREE_H +#define ERA_WRITESET_TREE_H #include "era/era_detail.h" #include "persistent-data/data-structures/btree.h" @@ -7,7 +7,7 @@ //---------------------------------------------------------------- namespace era { - namespace bloom_tree_detail { + namespace writeset_tree_detail { class damage_visitor; class damage { @@ -34,10 +34,10 @@ namespace era { run eras_; }; - struct damaged_bloom_filter : public damage { - damaged_bloom_filter(std::string const &desc, - uint32_t era, - run missing_bits); + struct damaged_writeset : public damage { + damaged_writeset(std::string const &desc, + uint32_t era, + run missing_bits); virtual void visit(damage_visitor &v) const; uint32_t era_; @@ -55,32 +55,31 @@ namespace era { } virtual void visit(missing_eras const &d) = 0; - virtual void visit(damaged_bloom_filter const &d) = 0; + virtual void visit(damaged_writeset const &d) = 0; }; - class bloom_visitor { + class writeset_visitor { public: - typedef boost::shared_ptr ptr; + typedef boost::shared_ptr ptr; - virtual ~bloom_visitor() {} + virtual ~writeset_visitor() {} - virtual void bloom_begin(uint32_t era, uint32_t nr_blocks, - uint32_t nr_bits, uint32_t nr_set) = 0; + virtual void writeset_begin(uint32_t era, uint32_t nr_bits) = 0; virtual void bit(uint32_t index, bool value) = 0; - virtual void bloom_end() = 0; + virtual void writeset_end() = 0; }; } - typedef persistent_data::btree<1, era_detail_traits> bloom_tree; + typedef persistent_data::btree<1, era_detail_traits> writeset_tree; - void walk_bloom_tree(persistent_data::transaction_manager::ptr tm, - bloom_tree const &tree, - bloom_tree_detail::bloom_visitor &bloom_v, - bloom_tree_detail::damage_visitor &dv); + void walk_writeset_tree(persistent_data::transaction_manager::ptr tm, + writeset_tree const &tree, + writeset_tree_detail::writeset_visitor &writeset_v, + writeset_tree_detail::damage_visitor &dv); - void check_bloom_tree(persistent_data::transaction_manager::ptr tm, - bloom_tree const &tree, - bloom_tree_detail::damage_visitor &dv); + void check_writeset_tree(persistent_data::transaction_manager::ptr tm, + writeset_tree const &tree, + writeset_tree_detail::damage_visitor &dv); } //---------------------------------------------------------------- diff --git a/era/xml_format.cc b/era/xml_format.cc index 9a63b8a..924a93b 100644 --- a/era/xml_format.cc +++ b/era/xml_format.cc @@ -33,25 +33,23 @@ namespace { out_ << "" << endl; } - void begin_bloom(uint32_t era, uint32_t nr_bits, - pd::block_address nr_blocks) { + void begin_writeset(uint32_t era, uint32_t nr_bits) { indent(); - out_ << "" << endl; + out_ << "" << endl; inc(); } - void bloom_bit(uint32_t bit, bool value) { + void writeset_bit(uint32_t bit, bool value) { indent(); // FIXME: collect all the bits, then uuencode out_ << "" << endl; } - void end_bloom() { + void end_writeset() { dec(); indent(); - out_ << "" << endl; + out_ << "" << endl; } void begin_era_array() { diff --git a/persistent-data/block.tcc b/persistent-data/block.tcc index 09cd313..431cdd0 100644 --- a/persistent-data/block.tcc +++ b/persistent-data/block.tcc @@ -343,7 +343,7 @@ namespace persistent_data { unsigned max_concurrent_blocks, typename block_io::mode mode) : io_(new block_io(path, nr_blocks, mode)), - cache_(max(64u, max_concurrent_blocks)), + cache_(max(1024u, max_concurrent_blocks)), tracker_(0, nr_blocks) { } diff --git a/unit-tests/array_t.cc b/unit-tests/array_t.cc index fa2dae1..0c688b4 100644 --- a/unit-tests/array_t.cc +++ b/unit-tests/array_t.cc @@ -20,6 +20,7 @@ #include "persistent-data/transaction_manager.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/data-structures/array.h" +#include "persistent-data/data-structures/simple_traits.h" #include diff --git a/unit-tests/bloom_filter_t.cc b/unit-tests/bloom_filter_t.cc index bb879ed..313355e 100644 --- a/unit-tests/bloom_filter_t.cc +++ b/unit-tests/bloom_filter_t.cc @@ -40,17 +40,26 @@ namespace { using namespace boost::random; - mt19937 rng; uniform_int_distribution uniform_dist(0, max); while (r.size() < count) { - block_address b = uniform_dist(rng); + block_address b = uniform_dist(rng_); r.insert(b); } return r; } + set generate_linear_blocks(unsigned count, + block_address max = std::numeric_limits::max()) { + set r; + + for (unsigned i = 0; i < count; i++) + r.insert(i); + + return r; + } + void commit() { block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK)); } @@ -65,6 +74,8 @@ namespace { block_manager<>::ptr bm_; space_map::ptr sm_; transaction_manager::ptr tm_; + + boost::random::mt19937 rng_; }; } @@ -119,18 +130,75 @@ TEST_F(BloomFilterTests, reload_works) } } -TEST_F(BloomFilterTests, count_false_positives) +unsigned next_power(unsigned n) { - block_address nr_blocks = 1024 * 1024; - block_address written_blocks = nr_blocks / 100; + unsigned r = 1; + while (r < n) + r <<= 1; - unsigned shift = 1; + return r; +} - while ((1ull << shift) < (16 * written_blocks)) - shift++; - cerr << "bitset size: " << ((1 << shift) / (8 * 1024)) << "k" << endl; +unsigned calc_nr_bits(double false_positive_rate, unsigned dirty_blocks_per_era, unsigned nr_probes) +{ + double k = (double) nr_probes; + double kth_root = exp(log(false_positive_rate) / k); // can be precomputed - bloom_filter f(tm_, 1 << shift, 6); + // FIXME: we need a way to calulate this in kernel? or should we + // just pass in the bloom params on the target line? + double tmp = log(1.0 - kth_root); + double n = (- k * (double) dirty_blocks_per_era) / tmp; + + return next_power(ceil(n)); +} + +unsigned calc_m(double fp, unsigned nr_probes, unsigned n) +{ + double k = (double) nr_probes; + double kth_root = exp(log(fp) / k); + double tmp = log(1.0 - kth_root); + double m = (- ((double) n) / k) * tmp; + + return ceil(m); +} + +void print_nr_bits_table(double fp, unsigned nr_probes) +{ + cout << "fp = " << fp << ", k = " << nr_probes << endl; + + for (unsigned long long m = 1024; m < (1ull << 25); m *= 2) { + unsigned n = calc_nr_bits(fp, m, nr_probes); + unsigned actual_m = calc_m(fp, nr_probes, n); + + cout << " m = " << m << ", n = " << n << ", " << n / (1024 * 8) + << "k, actual_m = " << actual_m << endl; + } +} + +// Not really a test +TEST_F(BloomFilterTests, nr_bits_table) +{ + print_nr_bits_table(0.001, 4); + print_nr_bits_table(0.001, 6); + print_nr_bits_table(0.001, 8); + print_nr_bits_table(0.001, 16); +} + +TEST_F(BloomFilterTests, count_false_positives_with_random_inserts) +{ + block_address nr_blocks = 1 << 27; + block_address written_blocks = nr_blocks / 1024; + + unsigned nr_probes = 6; + unsigned n = calc_nr_bits(0.001, written_blocks, nr_probes); + + cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl; + + double ideal_k = log(2) * ((double) n / (double) written_blocks); + cerr << "Ideal k = " << ideal_k << endl; + + + bloom_filter f(tm_, n, nr_probes); set bs = generate_random_blocks(written_blocks, nr_blocks); set::const_iterator it; @@ -138,8 +206,6 @@ TEST_F(BloomFilterTests, count_false_positives) for (it = bs.begin(); it != bs.end(); ++it) f.set(*it); - // f.print_debug(cerr); - unsigned count = 0; for (unsigned i = 0; i < nr_blocks; i++) if (!bs.count(i) && f.test(i)) @@ -150,4 +216,99 @@ TEST_F(BloomFilterTests, count_false_positives) << "%" << endl; } +TEST_F(BloomFilterTests, count_false_positives_with_linear_inserts) +{ + block_address nr_blocks = 1 << 25; + block_address written_blocks = nr_blocks / 100; + + double fp = 0.001; + unsigned nr_probes = 6; + unsigned n = calc_nr_bits(fp, written_blocks, nr_probes); + + cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl; + + double ideal_k = log(2) * ((double) n / (double) written_blocks); + cerr << "Ideal k = " << ideal_k << endl; + + + bloom_filter f(tm_, n, nr_probes); + + set bs = generate_linear_blocks(written_blocks, nr_blocks); + set::const_iterator it; + + for (it = bs.begin(); it != bs.end(); ++it) + f.set(*it); + + unsigned count = 0; + for (unsigned i = 0; i < nr_blocks; i++) + if (!bs.count(i) && f.test(i)) + count++; + + double actual_fp = static_cast(count) / static_cast(nr_blocks); + + ASSERT_THAT(actual_fp, Lt(fp)); + + cerr << count << " false positives out of " << nr_blocks << ", " + << actual_fp * 100.0 << "%" << endl; +} + +TEST_F(BloomFilterTests, false_positives_over_multiple_eras) +{ + unsigned nr_eras = 10; + block_address nr_blocks = 1 << 20; + block_address written_blocks = nr_blocks / nr_eras; + + double fp = 0.001; + unsigned nr_probes = 6; + unsigned n = calc_nr_bits(fp, written_blocks, nr_probes); + + cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl; + + double ideal_k = log(2) * ((double) n / (double) written_blocks); + cerr << "Ideal k = " << ideal_k << endl; + + vector > writes(nr_eras); + vector filters(nr_eras); + + for (unsigned era = 0; era < writes.size(); era++) { + cerr << "inserting era " << era << endl; + + writes[era] = generate_random_blocks(written_blocks, nr_blocks); + set const &bs = writes[era]; + + filters[era] = bloom_filter::ptr(new bloom_filter(tm_, n, nr_probes)); + bloom_filter::ptr &f = filters[era]; + + set::const_iterator it; + for (it = bs.begin(); it != bs.end(); ++it) + f->set(*it); + } + + set write_sum; + set filter_sum; + for (unsigned era_plus_1 = writes.size(); era_plus_1 > 0; era_plus_1--) { + unsigned era = era_plus_1 - 1; + + set const &era_writes = writes[era]; + write_sum.insert(era_writes.begin(), era_writes.end()); + + for (unsigned i = 0; i < nr_blocks; i++) + if (filters[era]->test(i)) + filter_sum.insert(i); + + unsigned count = 0; + for (unsigned i = 0; i < nr_blocks; i++) { + if (write_sum.count(i) > 0) + ASSERT_THAT(filter_sum.count(i), Gt(0ull)); + + else if (filter_sum.count(i)) + count++; + } + + cerr << "blocks >= era " << era << ", false positives = " + << static_cast(count * 100) / static_cast(nr_blocks) + << "%" << endl; + } +} + //---------------------------------------------------------------- diff --git a/unit-tests/btree_counter_t.cc b/unit-tests/btree_counter_t.cc index 5f4a7d0..adbf523 100644 --- a/unit-tests/btree_counter_t.cc +++ b/unit-tests/btree_counter_t.cc @@ -5,6 +5,7 @@ #include "persistent-data/data-structures/btree.h" #include "persistent-data/data-structures/btree_counter.h" #include "persistent-data/space-maps/core.h" +#include "persistent-data/data-structures/simple_traits.h" using namespace base; using namespace std; diff --git a/unit-tests/btree_t.cc b/unit-tests/btree_t.cc index b2f2647..8fe3a71 100644 --- a/unit-tests/btree_t.cc +++ b/unit-tests/btree_t.cc @@ -20,6 +20,7 @@ #include "persistent-data/transaction_manager.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/simple_traits.h" using namespace std; using namespace persistent_data;