emo
/
thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[era] switch from bloom filters to writesets

This commit is contained in:
Joe Thornber 2014-01-29 21:37:25 +00:00
parent e7bbfbc1fd
commit 8531a2befa
20 changed files with 439 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
Makefile.in
View File

@ -27,6 +27,7 @@ PROGRAMS=\
\
era_check \
era_dump \
era_invalidate \
\
thin_check \
thin_dump \
@ -53,7 +54,7 @@ SOURCE=\
era/era_array.cc \
era/era_detail.cc \
era/superblock.cc \
era/bloom_tree.cc \
era/writeset_tree.cc \
era/metadata.cc \
era/metadata_dump.cc \
era/xml_format.cc \
@ -305,7 +306,7 @@ ERA_CHECK_SOURCE=\
base/error_state.cc \
base/endian_utils.cc \
\
era/bloom_tree.cc \
era/writeset_tree.cc \
era/era_detail.cc \
era/era_array.cc \
era/metadata.cc \
@ -335,7 +336,7 @@ ERA_DUMP_SOURCE=\
base/error_state.cc \
base/endian_utils.cc \
\
era/bloom_tree.cc \
era/writeset_tree.cc \
era/era_detail.cc \
era/era_array.cc \
era/metadata.cc \
@ -362,6 +363,38 @@ era_dump: $(ERA_DUMP_OBJECTS) era/era_dump.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
ERA_INVALIDATE_SOURCE=\
base/base64.cc \
base/error_state.cc \
base/endian_utils.cc \
\
era/writeset_tree.cc \
era/era_detail.cc \
era/era_array.cc \
era/metadata.cc \
era/metadata_dump.cc \
era/superblock.cc \
era/xml_format.cc \
\
persistent-data/checksum.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
persistent-data/data-structures/bitset.cc \
persistent-data/space_map.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
ERA_INVALIDATE_OBJECTS=$(subst .cc,.o,$(ERA_INVALIDATE_SOURCE))
era_invalidate: $(ERA_INVALIDATE_OBJECTS) era/era_invalidate.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
#----------------------------------------------------------------
DEPEND_FILES=\

6
era/emitter.h
View File

@ -20,9 +20,9 @@ namespace era {
uint32_t current_era) = 0;
virtual void end_superblock() = 0;
virtual void begin_bloom(uint32_t era, uint32_t nr_bits, pd::block_address nr_blocks) = 0;
virtual void bloom_bit(uint32_t bit, bool value) = 0;
virtual void end_bloom() = 0;
virtual void begin_writeset(uint32_t era, uint32_t nr_bits) = 0;
virtual void writeset_bit(uint32_t bit, bool value) = 0;
virtual void end_writeset() = 0;
virtual void begin_era_array() = 0;
virtual void era(pd::block_address block, uint32_t era) = 0;

24
era/era_check.cc
View File

@ -14,7 +14,7 @@
#include "base/error_state.h"
#include "base/nested_output.h"
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "era/era_array.h"
#include "era/superblock.h"
#include "persistent-data/block.h"
@ -92,14 +92,14 @@ namespace {
using reporter_base::get_error;
};
class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base {
class writeset_tree_reporter : public writeset_tree_detail::damage_visitor, reporter_base {
public:
bloom_tree_reporter(nested_output &o)
writeset_tree_reporter(nested_output &o)
: reporter_base(o) {
}
void visit(bloom_tree_detail::missing_eras const &d) {
out() << "missing eras from bloom tree" << end_message();
void visit(writeset_tree_detail::missing_eras const &d) {
out() << "missing eras from writeset tree" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
@ -110,8 +110,8 @@ namespace {
mplus_error(FATAL);
}
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
out() << "damaged bloom filter" << end_message();
void visit(writeset_tree_detail::damaged_writeset const &d) {
out() << "damaged writeset" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
@ -216,22 +216,22 @@ namespace {
superblock sb = read_superblock(bm);
transaction_manager::ptr tm = open_tm(bm);
bloom_tree_reporter bt_rep(out);
writeset_tree_reporter wt_rep(out);
{
era_detail_traits::ref_counter rc(tm);
bloom_tree bt(tm, sb.bloom_tree_root, rc);
check_bloom_tree(tm, bt, bt_rep);
writeset_tree wt(tm, sb.writeset_tree_root, rc);
check_writeset_tree(tm, wt, wt_rep);
}
era_array_reporter ea_rep(out);
{
uint32_traits::ref_counter rc;
era_array ea(tm, rc, sb.era_array_root, sb.nr_blocks);
check_era_array(ea, ea_rep);
check_era_array(ea, sb.current_era, ea_rep);
}
return combine_errors(sb_rep.get_error(),
combine_errors(bt_rep.get_error(),
combine_errors(wt_rep.get_error(),
ea_rep.get_error()));
}

23
era/era_detail.cc
View File

@ -7,37 +7,18 @@ using namespace era;
//----------------------------------------------------------------
namespace {
#if 0
le32 pack_hash_detail(uint32_t hash1, uint32_t hash2, uint32_t nr_probes) {
throw std::runtime_error("not implemented");
}
void unpack_hash_detail(le32 packed, uint32_t &hash1, uint32_t &hash2, uint32_t &nr_probes) {
throw std::runtime_error("not implemented");
}
#endif
}
void
era_detail_traits::unpack(disk_type const &disk, value_type &value)
{
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
value.nr_bits = to_cpu<uint32_t>(disk.nr_bits);
value.nr_set = to_cpu<uint32_t>(disk.nr_set);
value.bloom_root = to_cpu<uint64_t>(disk.bloom_root);
//unpack_hash_detail(disk.hash_fns_and_probes, value.hash1, value.hash2, value.nr_probes);
value.writeset_root = to_cpu<uint64_t>(disk.writeset_root);
}
void
era_detail_traits::pack(value_type const &value, disk_type &disk)
{
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
disk.nr_bits = to_disk<le32>(value.nr_bits);
disk.nr_set = to_disk<le32>(value.nr_set);
disk.bloom_root = to_disk<le64>(value.bloom_root);
// disk.hash_fns_and_probes = pack_hash_detail(value.hash1, value.hash2, value.nr_probes);
disk.writeset_root = to_disk<le64>(value.writeset_root);
}
//----------------------------------------------------------------

13
era/era_detail.h
View File

@ -8,22 +8,13 @@
namespace era {
struct era_detail_disk {
base::le32 nr_blocks;
base::le32 nr_bits;
base::le32 nr_set;
base::le64 bloom_root;
base::le64 writeset_root;
} __attribute__ ((packed));
struct era_detail {
uint32_t nr_blocks;
uint32_t nr_bits;
uint32_t nr_set;
uint32_t hash1;
uint32_t hash2;
uint32_t nr_probes;
uint64_t bloom_root;
uint64_t writeset_root;
};
// FIXME: implement

2
era/era_dump.cc
View File

@ -5,7 +5,7 @@
#include "version.h"
#include "era/era_array.h"
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "era/metadata.h"
#include "era/metadata_dump.h"
#include "era/xml_format.h"

114
era/era_invalidate.cc Normal file
View File

@ -0,0 +1,114 @@
#include <fstream>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include "version.h"
#include "era/era_array.h"
#include "era/writeset_tree.h"
#include "era/metadata.h"
#include "era/xml_format.h"
#include "persistent-data/file_utils.h"
using namespace era;
using namespace std;
//----------------------------------------------------------------
namespace {
struct flags {
flags() {
}
};
//--------------------------------
void emit_blocks() {
}
//--------------------------------
string const STDOUT_PATH("-");
bool want_stdout(string const &output) {
return output == STDOUT_PATH;
}
int invalidate(string const &dev, string const &output, flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY);
metadata::ptr md(new metadata(bm, metadata::OPEN));
if (want_stdout(output)) {
emitter::ptr e = create_xml_emitter(cout);
//emit_blocks(md, e, fs);
} else {
ofstream out(output.c_str());
emitter::ptr e = create_xml_emitter(out);
//emit_blocks(md, e, fs);
}
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-o <xml file>}" << endl
<< " {-V|--version}" << endl;
}
}
//----------------------------------------------------------------
int main(int argc, char **argv)
{
int c;
flags fs;
string output("-");
char const shortopts[] = "ho:V";
option const longopts[] = {
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout, basename(argv[0]));
return 0;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, basename(argv[0]));
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
return invalidate(argv[optind], output, fs);
}
//----------------------------------------------------------------

6
era/metadata.cc
View File

@ -47,9 +47,9 @@ metadata::open_metadata(block_manager<>::ptr bm)
tm_ = open_tm(bm);
sb_ = read_superblock(tm_->get_bm());
bloom_tree_ = bloom_tree::ptr(new bloom_tree(tm_,
sb_.bloom_tree_root,
era_detail_traits::ref_counter(tm_)));
writeset_tree_ = writeset_tree::ptr(new writeset_tree(tm_,
sb_.writeset_tree_root,
era_detail_traits::ref_counter(tm_)));
era_array_ = era_array::ptr(new era_array(tm_,
uint32_traits::ref_counter(),

4
era/metadata.h
View File

@ -10,7 +10,7 @@
#include "persistent-data/transaction_manager.h"
#include "era/superblock.h"
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "era/era_array.h"
//----------------------------------------------------------------
@ -34,7 +34,7 @@ namespace era {
tm::ptr tm_;
superblock sb_;
checked_space_map::ptr metadata_sm_;
bloom_tree::ptr bloom_tree_;
writeset_tree::ptr writeset_tree_;
era_array::ptr era_array_;
private:

40
era/metadata_dump.cc
View File

@ -17,42 +17,42 @@ namespace {
"perhaps you wanted to run with --repair ?");
}
class bloom_tree_emitter : public bloom_tree_detail::bloom_visitor {
class writeset_tree_emitter : public writeset_tree_detail::writeset_visitor {
public:
bloom_tree_emitter(emitter::ptr e)
writeset_tree_emitter(emitter::ptr e)
: e_(e) {
}
virtual void bloom_begin(uint32_t era, uint32_t nr_blocks, uint32_t nr_bits, uint32_t nr_set) {
e_->begin_bloom(era, nr_bits, nr_blocks);
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) {
e_->begin_writeset(era, nr_bits);
}
virtual void bit(uint32_t bit, bool value) {
e_->bloom_bit(bit, value);
e_->writeset_bit(bit, value);
}
virtual void bloom_end() {
e_->end_bloom();
virtual void writeset_end() {
e_->end_writeset();
}
private:
emitter::ptr e_;
};
struct ignore_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
void visit(bloom_tree_detail::missing_eras const &d) {
struct ignore_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
}
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
void visit(writeset_tree_detail::damaged_writeset const &d) {
}
};
struct fatal_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
void visit(bloom_tree_detail::missing_eras const &d) {
struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
raise_metadata_damage();
}
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
void visit(writeset_tree_detail::damaged_writeset const &d) {
raise_metadata_damage();
}
};
@ -104,15 +104,15 @@ era::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair)
sb.current_era);
{
{
bloom_tree_emitter visitor(e);
writeset_tree_emitter visitor(e);
ignore_bloom_tree_damage ignore;
fatal_bloom_tree_damage fatal;
bloom_tree_detail::damage_visitor &dv = repair ?
static_cast<bloom_tree_detail::damage_visitor &>(ignore) :
static_cast<bloom_tree_detail::damage_visitor &>(fatal);
ignore_writeset_tree_damage ignore;
fatal_writeset_tree_damage fatal;
writeset_tree_detail::damage_visitor &dv = repair ?
static_cast<writeset_tree_detail::damage_visitor &>(ignore) :
static_cast<writeset_tree_detail::damage_visitor &>(fatal);
walk_bloom_tree(md->tm_, *md->bloom_tree_, visitor, dv);
walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv);
}
e->begin_era_array();

16
era/superblock.cc
View File

@ -34,7 +34,7 @@ namespace {
le32 current_era;
era_detail_disk current_detail;
le64 bloom_tree_root;
le64 writeset_tree_root;
le64 era_array_root;
} __attribute__ ((packed));
@ -116,7 +116,7 @@ superblock::superblock()
metadata_block_size(8),
nr_blocks(0),
current_era(0),
bloom_tree_root(0),
writeset_tree_root(0),
era_array_root(0)
{
memset(uuid, 0, sizeof(uuid));
@ -141,7 +141,7 @@ superblock_traits::unpack(disk_type const &disk, value_type &value)
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
value.current_era = to_cpu<uint32_t>(disk.current_era);
era_detail_traits::unpack(disk.current_detail, value.current_detail);
value.bloom_tree_root = to_cpu<uint64_t>(disk.bloom_tree_root);
value.writeset_tree_root = to_cpu<uint64_t>(disk.writeset_tree_root);
value.era_array_root = to_cpu<uint64_t>(disk.era_array_root);
}
@ -161,7 +161,7 @@ superblock_traits::pack(value_type const &value, disk_type &disk)
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
disk.current_era = to_disk<le32>(value.current_era);
era_detail_traits::pack(value.current_detail, disk.current_detail);
disk.bloom_tree_root = to_disk<le64>(value.bloom_tree_root);
disk.writeset_tree_root = to_disk<le64>(value.writeset_tree_root);
disk.era_array_root = to_disk<le64>(value.era_array_root);
}
@ -282,8 +282,8 @@ era::check_superblock(superblock const &sb,
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) {
string msg("bloom tree root points back to the superblock");
if (sb.writeset_tree_root == SUPERBLOCK_LOCATION) {
string msg("writeset tree root points back to the superblock");
visitor.visit(superblock_invalid(msg));
}
@ -292,9 +292,9 @@ era::check_superblock(superblock const &sb,
visitor.visit(superblock_invalid(msg));
}
if (sb.bloom_tree_root == sb.era_array_root) {
if (sb.writeset_tree_root == sb.era_array_root) {
ostringstream msg;
msg << "bloom tree root and era array both point to the same block: "
msg << "writeset tree root and era array both point to the same block: "
<< sb.era_array_root;
visitor.visit(superblock_invalid(msg.str()));
}

2
era/superblock.h
View File

@ -59,7 +59,7 @@ namespace era {
era_detail current_detail;
// A btree of undigested era_details
uint64_t bloom_tree_root;
uint64_t writeset_tree_root;
// Big array holding the digested era/block info.
uint64_t era_array_root;

63
era/bloom_tree.cc → era/writeset_tree.cc
View File

@ -1,10 +1,10 @@
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include "persistent-data/data-structures/bitset.h"
using namespace boost;
using namespace era;
using namespace bloom_tree_detail;
using namespace writeset_tree_detail;
using namespace persistent_data;
using namespace std;
@ -22,9 +22,9 @@ missing_eras::visit(damage_visitor &v) const {
v.visit(*this);
}
damaged_bloom_filter::damaged_bloom_filter(string const &desc,
uint32_t era,
run<uint32_t> missing_bits)
damaged_writeset::damaged_writeset(string const &desc,
uint32_t era,
run<uint32_t> missing_bits)
: damage(desc),
era_(era),
missing_bits_(missing_bits)
@ -32,7 +32,7 @@ damaged_bloom_filter::damaged_bloom_filter(string const &desc,
}
void
damaged_bloom_filter::visit(damage_visitor &v) const
damaged_writeset::visit(damage_visitor &v) const
{
v.visit(*this);
}
@ -40,39 +40,39 @@ damaged_bloom_filter::visit(damage_visitor &v) const
//----------------------------------------------------------------
namespace {
class ll_bloom_visitor : public bitset_detail::bitset_visitor {
class ll_writeset_visitor : public bitset_detail::bitset_visitor {
public:
typedef persistent_data::transaction_manager::ptr tm_ptr;
ll_bloom_visitor(tm_ptr tm,
bloom_tree_detail::bloom_visitor &bloom_v,
bloom_tree_detail::damage_visitor &dv)
ll_writeset_visitor(tm_ptr tm,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv)
: tm_(tm),
bloom_v_(bloom_v),
writeset_v_(writeset_v),
dv_(dv) {
}
void visit(btree_path const &path, era_detail const &era) {
era_ = path[0];
bitset bs(tm_, era.bloom_root, era.nr_bits);
bloom_v_.bloom_begin(era_, era.nr_blocks, era.nr_bits, era.nr_set);
bitset bs(tm_, era.writeset_root, era.nr_bits);
writeset_v_.writeset_begin(era_, era.nr_bits);
bs.walk_bitset(*this);
bloom_v_.bloom_end();
writeset_v_.writeset_end();
}
void visit(uint32_t index, bool value) {
bloom_v_.bit(index, value);
writeset_v_.bit(index, value);
}
void visit(bitset_detail::missing_bits const &d) {
dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_));
dv_.visit(writeset_tree_detail::damaged_writeset("missing bits", era_, d.keys_));
}
private:
tm_ptr tm_;
uint64_t era_;
bloom_tree_detail::bloom_visitor &bloom_v_;
bloom_tree_detail::damage_visitor &dv_;
writeset_tree_detail::writeset_visitor &writeset_v_;
writeset_tree_detail::damage_visitor &dv_;
};
class ll_damage_visitor {
@ -98,38 +98,37 @@ namespace {
}
void
era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::bloom_visitor &bloom_v,
bloom_tree_detail::damage_visitor &dv)
era::walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv)
{
ll_bloom_visitor ll_bv(tm, bloom_v, dv);
ll_writeset_visitor ll_bv(tm, writeset_v, dv);
ll_damage_visitor ll_dv(dv);
btree_visit_values(tree, ll_bv, ll_dv);
}
namespace {
class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor {
class noop_writeset_visitor : public writeset_tree_detail::writeset_visitor {
public:
void bloom_begin(uint32_t era, uint32_t nr_blocks,
uint32_t nr_bits, uint32_t nr_set) {
void writeset_begin(uint32_t era, uint32_t nr_bits) {
}
void bit(uint32_t index, bool value) {
}
void bloom_end() {
void writeset_end() {
}
};
};
void
era::check_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::damage_visitor &dv)
era::check_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::damage_visitor &dv)
{
noop_bloom_visitor bv;
walk_bloom_tree(tm, tree, bv, dv);
noop_writeset_visitor bv;
walk_writeset_tree(tm, tree, bv, dv);
}
//----------------------------------------------------------------

43
era/bloom_tree.h → era/writeset_tree.h
View File

@ -1,5 +1,5 @@
#ifndef ERA_BLOOM_TREE_H
#define ERA_BLOOM_TREE_H
#ifndef ERA_WRITESET_TREE_H
#define ERA_WRITESET_TREE_H
#include "era/era_detail.h"
#include "persistent-data/data-structures/btree.h"
@ -7,7 +7,7 @@
//----------------------------------------------------------------
namespace era {
namespace bloom_tree_detail {
namespace writeset_tree_detail {
class damage_visitor;
class damage {
@ -34,10 +34,10 @@ namespace era {
run<uint32_t> eras_;
};
struct damaged_bloom_filter : public damage {
damaged_bloom_filter(std::string const &desc,
uint32_t era,
run<uint32_t> missing_bits);
struct damaged_writeset : public damage {
damaged_writeset(std::string const &desc,
uint32_t era,
run<uint32_t> missing_bits);
virtual void visit(damage_visitor &v) const;
uint32_t era_;
@ -55,32 +55,31 @@ namespace era {
}
virtual void visit(missing_eras const &d) = 0;
virtual void visit(damaged_bloom_filter const &d) = 0;
virtual void visit(damaged_writeset const &d) = 0;
};
class bloom_visitor {
class writeset_visitor {
public:
typedef boost::shared_ptr<bloom_visitor> ptr;
typedef boost::shared_ptr<writeset_visitor> ptr;
virtual ~bloom_visitor() {}
virtual ~writeset_visitor() {}
virtual void bloom_begin(uint32_t era, uint32_t nr_blocks,
uint32_t nr_bits, uint32_t nr_set) = 0;
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) = 0;
virtual void bit(uint32_t index, bool value) = 0;
virtual void bloom_end() = 0;
virtual void writeset_end() = 0;
};
}
typedef persistent_data::btree<1, era_detail_traits> bloom_tree;
typedef persistent_data::btree<1, era_detail_traits> writeset_tree;
void walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::bloom_visitor &bloom_v,
bloom_tree_detail::damage_visitor &dv);
void walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv);
void check_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::damage_visitor &dv);
void check_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::damage_visitor &dv);
}
//----------------------------------------------------------------

14
era/xml_format.cc
View File

@ -33,25 +33,23 @@ namespace {
out_ << "</superblock>" << endl;
}
void begin_bloom(uint32_t era, uint32_t nr_bits,
pd::block_address nr_blocks) {
void begin_writeset(uint32_t era, uint32_t nr_bits) {
indent();
out_ << "<bloom era=\"" << era << "\""
<< " nr_bits=\"" << nr_bits << "\""
<< " nr_blocks=\"" << nr_blocks << "\">" << endl;
out_ << "<writeset era=\"" << era << "\""
<< " nr_bits=\"" << nr_bits << "\">" << endl;
inc();
}
void bloom_bit(uint32_t bit, bool value) {
void writeset_bit(uint32_t bit, bool value) {
indent();
// FIXME: collect all the bits, then uuencode
out_ << "<bit bit=\"" << bit << "\" value=\"" << value << "\">" << endl;
}
void end_bloom() {
void end_writeset() {
dec();
indent();
out_ << "</bloom>" << endl;
out_ << "</writeset>" << endl;
}
void begin_era_array() {

2
persistent-data/block.tcc
View File

@ -343,7 +343,7 @@ namespace persistent_data {
unsigned max_concurrent_blocks,
typename block_io<BlockSize>::mode mode)
: io_(new block_io<BlockSize>(path, nr_blocks, mode)),
cache_(max(64u, max_concurrent_blocks)),
cache_(max(1024u, max_concurrent_blocks)),
tracker_(0, nr_blocks)
{
}

1
unit-tests/array_t.cc
View File

@ -20,6 +20,7 @@
#include "persistent-data/transaction_manager.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/data-structures/array.h"
#include "persistent-data/data-structures/simple_traits.h"
#include <vector>

185
unit-tests/bloom_filter_t.cc
View File

@ -40,17 +40,26 @@ namespace {
using namespace boost::random;
mt19937 rng;
uniform_int_distribution<uint64_t> uniform_dist(0, max);
while (r.size() < count) {
block_address b = uniform_dist(rng);
block_address b = uniform_dist(rng_);
r.insert(b);
}
return r;
}
set<block_address> generate_linear_blocks(unsigned count,
block_address max = std::numeric_limits<uint64_t>::max()) {
set<block_address> r;
for (unsigned i = 0; i < count; i++)
r.insert(i);
return r;
}
void commit() {
block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK));
}
@ -65,6 +74,8 @@ namespace {
block_manager<>::ptr bm_;
space_map::ptr sm_;
transaction_manager::ptr tm_;
boost::random::mt19937 rng_;
};
}
@ -119,18 +130,75 @@ TEST_F(BloomFilterTests, reload_works)
}
}
TEST_F(BloomFilterTests, count_false_positives)
unsigned next_power(unsigned n)
{
block_address nr_blocks = 1024 * 1024;
block_address written_blocks = nr_blocks / 100;
unsigned r = 1;
while (r < n)
r <<= 1;
unsigned shift = 1;
return r;
}
while ((1ull << shift) < (16 * written_blocks))
shift++;
cerr << "bitset size: " << ((1 << shift) / (8 * 1024)) << "k" << endl;
unsigned calc_nr_bits(double false_positive_rate, unsigned dirty_blocks_per_era, unsigned nr_probes)
{
double k = (double) nr_probes;
double kth_root = exp(log(false_positive_rate) / k); // can be precomputed
bloom_filter f(tm_, 1 << shift, 6);
// FIXME: we need a way to calulate this in kernel? or should we
// just pass in the bloom params on the target line?
double tmp = log(1.0 - kth_root);
double n = (- k * (double) dirty_blocks_per_era) / tmp;
return next_power(ceil(n));
}
unsigned calc_m(double fp, unsigned nr_probes, unsigned n)
{
double k = (double) nr_probes;
double kth_root = exp(log(fp) / k);
double tmp = log(1.0 - kth_root);
double m = (- ((double) n) / k) * tmp;
return ceil(m);
}
void print_nr_bits_table(double fp, unsigned nr_probes)
{
cout << "fp = " << fp << ", k = " << nr_probes << endl;
for (unsigned long long m = 1024; m < (1ull << 25); m *= 2) {
unsigned n = calc_nr_bits(fp, m, nr_probes);
unsigned actual_m = calc_m(fp, nr_probes, n);
cout << " m = " << m << ", n = " << n << ", " << n / (1024 * 8)
<< "k, actual_m = " << actual_m << endl;
}
}
// Not really a test
TEST_F(BloomFilterTests, nr_bits_table)
{
print_nr_bits_table(0.001, 4);
print_nr_bits_table(0.001, 6);
print_nr_bits_table(0.001, 8);
print_nr_bits_table(0.001, 16);
}
TEST_F(BloomFilterTests, count_false_positives_with_random_inserts)
{
block_address nr_blocks = 1 << 27;
block_address written_blocks = nr_blocks / 1024;
unsigned nr_probes = 6;
unsigned n = calc_nr_bits(0.001, written_blocks, nr_probes);
cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
double ideal_k = log(2) * ((double) n / (double) written_blocks);
cerr << "Ideal k = " << ideal_k << endl;
bloom_filter f(tm_, n, nr_probes);
set<block_address> bs = generate_random_blocks(written_blocks, nr_blocks);
set<block_address>::const_iterator it;
@ -138,8 +206,6 @@ TEST_F(BloomFilterTests, count_false_positives)
for (it = bs.begin(); it != bs.end(); ++it)
f.set(*it);
// f.print_debug(cerr);
unsigned count = 0;
for (unsigned i = 0; i < nr_blocks; i++)
if (!bs.count(i) && f.test(i))
@ -150,4 +216,99 @@ TEST_F(BloomFilterTests, count_false_positives)
<< "%" << endl;
}
TEST_F(BloomFilterTests, count_false_positives_with_linear_inserts)
{
block_address nr_blocks = 1 << 25;
block_address written_blocks = nr_blocks / 100;
double fp = 0.001;
unsigned nr_probes = 6;
unsigned n = calc_nr_bits(fp, written_blocks, nr_probes);
cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
double ideal_k = log(2) * ((double) n / (double) written_blocks);
cerr << "Ideal k = " << ideal_k << endl;
bloom_filter f(tm_, n, nr_probes);
set<block_address> bs = generate_linear_blocks(written_blocks, nr_blocks);
set<block_address>::const_iterator it;
for (it = bs.begin(); it != bs.end(); ++it)
f.set(*it);
unsigned count = 0;
for (unsigned i = 0; i < nr_blocks; i++)
if (!bs.count(i) && f.test(i))
count++;
double actual_fp = static_cast<double>(count) / static_cast<double>(nr_blocks);
ASSERT_THAT(actual_fp, Lt(fp));
cerr << count << " false positives out of " << nr_blocks << ", "
<< actual_fp * 100.0 << "%" << endl;
}
TEST_F(BloomFilterTests, false_positives_over_multiple_eras)
{
unsigned nr_eras = 10;
block_address nr_blocks = 1 << 20;
block_address written_blocks = nr_blocks / nr_eras;
double fp = 0.001;
unsigned nr_probes = 6;
unsigned n = calc_nr_bits(fp, written_blocks, nr_probes);
cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
double ideal_k = log(2) * ((double) n / (double) written_blocks);
cerr << "Ideal k = " << ideal_k << endl;
vector<set<block_address> > writes(nr_eras);
vector<bloom_filter::ptr> filters(nr_eras);
for (unsigned era = 0; era < writes.size(); era++) {
cerr << "inserting era " << era << endl;
writes[era] = generate_random_blocks(written_blocks, nr_blocks);
set<block_address> const &bs = writes[era];
filters[era] = bloom_filter::ptr(new bloom_filter(tm_, n, nr_probes));
bloom_filter::ptr &f = filters[era];
set<block_address>::const_iterator it;
for (it = bs.begin(); it != bs.end(); ++it)
f->set(*it);
}
set<block_address> write_sum;
set<block_address> filter_sum;
for (unsigned era_plus_1 = writes.size(); era_plus_1 > 0; era_plus_1--) {
unsigned era = era_plus_1 - 1;
set<block_address> const &era_writes = writes[era];
write_sum.insert(era_writes.begin(), era_writes.end());
for (unsigned i = 0; i < nr_blocks; i++)
if (filters[era]->test(i))
filter_sum.insert(i);
unsigned count = 0;
for (unsigned i = 0; i < nr_blocks; i++) {
if (write_sum.count(i) > 0)
ASSERT_THAT(filter_sum.count(i), Gt(0ull));
else if (filter_sum.count(i))
count++;
}
cerr << "blocks >= era " << era << ", false positives = "
<< static_cast<double>(count * 100) / static_cast<double>(nr_blocks)
<< "%" << endl;
}
}
//----------------------------------------------------------------

1
unit-tests/btree_counter_t.cc
View File

@ -5,6 +5,7 @@
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/btree_counter.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/data-structures/simple_traits.h"
using namespace base;
using namespace std;

1
unit-tests/btree_t.cc
View File

@ -20,6 +20,7 @@
#include "persistent-data/transaction_manager.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/simple_traits.h"
using namespace std;
using namespace persistent_data;