emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[era] switch from bloom filters to writesets

Browse Source
This commit is contained in:
Joe Thornber
2014-01-29 21:37:25 +00:00
parent e7bbfbc1fd
commit 8531a2befa
20 changed files with 439 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
era/emitter.h
View File

@@ -20,9 +20,9 @@ namespace era {
uint32_t current_era) = 0;
virtual void end_superblock() = 0;
virtual void begin_bloom(uint32_t era, uint32_t nr_bits, pd::block_address nr_blocks) = 0;
virtual void bloom_bit(uint32_t bit, bool value) = 0;
virtual void end_bloom() = 0;
virtual void begin_writeset(uint32_t era, uint32_t nr_bits) = 0;
virtual void writeset_bit(uint32_t bit, bool value) = 0;
virtual void end_writeset() = 0;
virtual void begin_era_array() = 0;
virtual void era(pd::block_address block, uint32_t era) = 0;

24
era/era_check.cc
View File

@@ -14,7 +14,7 @@
#include "base/error_state.h"
#include "base/nested_output.h"
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "era/era_array.h"
#include "era/superblock.h"
#include "persistent-data/block.h"
@@ -92,14 +92,14 @@ namespace {
using reporter_base::get_error;
};
class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base {
class writeset_tree_reporter : public writeset_tree_detail::damage_visitor, reporter_base {
public:
bloom_tree_reporter(nested_output &o)
writeset_tree_reporter(nested_output &o)
: reporter_base(o) {
}
void visit(bloom_tree_detail::missing_eras const &d) {
out() << "missing eras from bloom tree" << end_message();
void visit(writeset_tree_detail::missing_eras const &d) {
out() << "missing eras from writeset tree" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
@@ -110,8 +110,8 @@ namespace {
mplus_error(FATAL);
}
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
out() << "damaged bloom filter" << end_message();
void visit(writeset_tree_detail::damaged_writeset const &d) {
out() << "damaged writeset" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
@@ -216,22 +216,22 @@ namespace {
superblock sb = read_superblock(bm);
transaction_manager::ptr tm = open_tm(bm);
bloom_tree_reporter bt_rep(out);
writeset_tree_reporter wt_rep(out);
{
era_detail_traits::ref_counter rc(tm);
bloom_tree bt(tm, sb.bloom_tree_root, rc);
check_bloom_tree(tm, bt, bt_rep);
writeset_tree wt(tm, sb.writeset_tree_root, rc);
check_writeset_tree(tm, wt, wt_rep);
}
era_array_reporter ea_rep(out);
{
uint32_traits::ref_counter rc;
era_array ea(tm, rc, sb.era_array_root, sb.nr_blocks);
check_era_array(ea, ea_rep);
check_era_array(ea, sb.current_era, ea_rep);
}
return combine_errors(sb_rep.get_error(),
combine_errors(bt_rep.get_error(),
combine_errors(wt_rep.get_error(),
ea_rep.get_error()));
}

23
era/era_detail.cc
View File

@@ -7,37 +7,18 @@ using namespace era;
//----------------------------------------------------------------
namespace {
#if 0
le32 pack_hash_detail(uint32_t hash1, uint32_t hash2, uint32_t nr_probes) {
throw std::runtime_error("not implemented");
}
void unpack_hash_detail(le32 packed, uint32_t &hash1, uint32_t &hash2, uint32_t &nr_probes) {
throw std::runtime_error("not implemented");
}
#endif
}
void
era_detail_traits::unpack(disk_type const &disk, value_type &value)
{
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
value.nr_bits = to_cpu<uint32_t>(disk.nr_bits);
value.nr_set = to_cpu<uint32_t>(disk.nr_set);
value.bloom_root = to_cpu<uint64_t>(disk.bloom_root);
//unpack_hash_detail(disk.hash_fns_and_probes, value.hash1, value.hash2, value.nr_probes);
value.writeset_root = to_cpu<uint64_t>(disk.writeset_root);
}
void
era_detail_traits::pack(value_type const &value, disk_type &disk)
{
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
disk.nr_bits = to_disk<le32>(value.nr_bits);
disk.nr_set = to_disk<le32>(value.nr_set);
disk.bloom_root = to_disk<le64>(value.bloom_root);
// disk.hash_fns_and_probes = pack_hash_detail(value.hash1, value.hash2, value.nr_probes);
disk.writeset_root = to_disk<le64>(value.writeset_root);
}
//----------------------------------------------------------------

13
era/era_detail.h
View File

@@ -8,22 +8,13 @@
namespace era {
struct era_detail_disk {
base::le32 nr_blocks;
base::le32 nr_bits;
base::le32 nr_set;
base::le64 bloom_root;
base::le64 writeset_root;
} __attribute__ ((packed));
struct era_detail {
uint32_t nr_blocks;
uint32_t nr_bits;
uint32_t nr_set;
uint32_t hash1;
uint32_t hash2;
uint32_t nr_probes;
uint64_t bloom_root;
uint64_t writeset_root;
};
// FIXME: implement

2
era/era_dump.cc
View File

@@ -5,7 +5,7 @@
#include "version.h"
#include "era/era_array.h"
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "era/metadata.h"
#include "era/metadata_dump.h"
#include "era/xml_format.h"

114
era/era_invalidate.cc Normal file
View File

@@ -0,0 +1,114 @@
#include <fstream>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include "version.h"
#include "era/era_array.h"
#include "era/writeset_tree.h"
#include "era/metadata.h"
#include "era/xml_format.h"
#include "persistent-data/file_utils.h"
using namespace era;
using namespace std;
//----------------------------------------------------------------
namespace {
struct flags {
flags() {
}
};
//--------------------------------
void emit_blocks() {
}
//--------------------------------
string const STDOUT_PATH("-");
bool want_stdout(string const &output) {
return output == STDOUT_PATH;
}
int invalidate(string const &dev, string const &output, flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY);
metadata::ptr md(new metadata(bm, metadata::OPEN));
if (want_stdout(output)) {
emitter::ptr e = create_xml_emitter(cout);
//emit_blocks(md, e, fs);
} else {
ofstream out(output.c_str());
emitter::ptr e = create_xml_emitter(out);
//emit_blocks(md, e, fs);
}
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-o <xml file>}" << endl
<< " {-V|--version}" << endl;
}
}
//----------------------------------------------------------------
int main(int argc, char **argv)
{
int c;
flags fs;
string output("-");
char const shortopts[] = "ho:V";
option const longopts[] = {
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout, basename(argv[0]));
return 0;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, basename(argv[0]));
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
return invalidate(argv[optind], output, fs);
}
//----------------------------------------------------------------

6
era/metadata.cc
View File

@@ -47,9 +47,9 @@ metadata::open_metadata(block_manager<>::ptr bm)
tm_ = open_tm(bm);
sb_ = read_superblock(tm_->get_bm());
bloom_tree_ = bloom_tree::ptr(new bloom_tree(tm_,
sb_.bloom_tree_root,
era_detail_traits::ref_counter(tm_)));
writeset_tree_ = writeset_tree::ptr(new writeset_tree(tm_,
sb_.writeset_tree_root,
era_detail_traits::ref_counter(tm_)));
era_array_ = era_array::ptr(new era_array(tm_,
uint32_traits::ref_counter(),

4
era/metadata.h
View File

@@ -10,7 +10,7 @@
#include "persistent-data/transaction_manager.h"
#include "era/superblock.h"
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "era/era_array.h"
//----------------------------------------------------------------
@@ -34,7 +34,7 @@ namespace era {
tm::ptr tm_;
superblock sb_;
checked_space_map::ptr metadata_sm_;
bloom_tree::ptr bloom_tree_;
writeset_tree::ptr writeset_tree_;
era_array::ptr era_array_;
private:

40
era/metadata_dump.cc
View File

@@ -17,42 +17,42 @@ namespace {
"perhaps you wanted to run with --repair ?");
}
class bloom_tree_emitter : public bloom_tree_detail::bloom_visitor {
class writeset_tree_emitter : public writeset_tree_detail::writeset_visitor {
public:
bloom_tree_emitter(emitter::ptr e)
writeset_tree_emitter(emitter::ptr e)
: e_(e) {
}
virtual void bloom_begin(uint32_t era, uint32_t nr_blocks, uint32_t nr_bits, uint32_t nr_set) {
e_->begin_bloom(era, nr_bits, nr_blocks);
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) {
e_->begin_writeset(era, nr_bits);
}
virtual void bit(uint32_t bit, bool value) {
e_->bloom_bit(bit, value);
e_->writeset_bit(bit, value);
}
virtual void bloom_end() {
e_->end_bloom();
virtual void writeset_end() {
e_->end_writeset();
}
private:
emitter::ptr e_;
};
struct ignore_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
void visit(bloom_tree_detail::missing_eras const &d) {
struct ignore_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
}
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
void visit(writeset_tree_detail::damaged_writeset const &d) {
}
};
struct fatal_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
void visit(bloom_tree_detail::missing_eras const &d) {
struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
raise_metadata_damage();
}
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
void visit(writeset_tree_detail::damaged_writeset const &d) {
raise_metadata_damage();
}
};
@@ -104,15 +104,15 @@ era::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair)
sb.current_era);
{
{
bloom_tree_emitter visitor(e);
writeset_tree_emitter visitor(e);
ignore_bloom_tree_damage ignore;
fatal_bloom_tree_damage fatal;
bloom_tree_detail::damage_visitor &dv = repair ?
static_cast<bloom_tree_detail::damage_visitor &>(ignore) :
static_cast<bloom_tree_detail::damage_visitor &>(fatal);
ignore_writeset_tree_damage ignore;
fatal_writeset_tree_damage fatal;
writeset_tree_detail::damage_visitor &dv = repair ?
static_cast<writeset_tree_detail::damage_visitor &>(ignore) :
static_cast<writeset_tree_detail::damage_visitor &>(fatal);
walk_bloom_tree(md->tm_, *md->bloom_tree_, visitor, dv);
walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv);
}
e->begin_era_array();

16
era/superblock.cc
View File

@@ -34,7 +34,7 @@ namespace {
le32 current_era;
era_detail_disk current_detail;
le64 bloom_tree_root;
le64 writeset_tree_root;
le64 era_array_root;
} __attribute__ ((packed));
@@ -116,7 +116,7 @@ superblock::superblock()
metadata_block_size(8),
nr_blocks(0),
current_era(0),
bloom_tree_root(0),
writeset_tree_root(0),
era_array_root(0)
{
memset(uuid, 0, sizeof(uuid));
@@ -141,7 +141,7 @@ superblock_traits::unpack(disk_type const &disk, value_type &value)
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
value.current_era = to_cpu<uint32_t>(disk.current_era);
era_detail_traits::unpack(disk.current_detail, value.current_detail);
value.bloom_tree_root = to_cpu<uint64_t>(disk.bloom_tree_root);
value.writeset_tree_root = to_cpu<uint64_t>(disk.writeset_tree_root);
value.era_array_root = to_cpu<uint64_t>(disk.era_array_root);
}
@@ -161,7 +161,7 @@ superblock_traits::pack(value_type const &value, disk_type &disk)
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
disk.current_era = to_disk<le32>(value.current_era);
era_detail_traits::pack(value.current_detail, disk.current_detail);
disk.bloom_tree_root = to_disk<le64>(value.bloom_tree_root);
disk.writeset_tree_root = to_disk<le64>(value.writeset_tree_root);
disk.era_array_root = to_disk<le64>(value.era_array_root);
}
@@ -282,8 +282,8 @@ era::check_superblock(superblock const &sb,
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) {
string msg("bloom tree root points back to the superblock");
if (sb.writeset_tree_root == SUPERBLOCK_LOCATION) {
string msg("writeset tree root points back to the superblock");
visitor.visit(superblock_invalid(msg));
}
@@ -292,9 +292,9 @@ era::check_superblock(superblock const &sb,
visitor.visit(superblock_invalid(msg));
}
if (sb.bloom_tree_root == sb.era_array_root) {
if (sb.writeset_tree_root == sb.era_array_root) {
ostringstream msg;
msg << "bloom tree root and era array both point to the same block: "
msg << "writeset tree root and era array both point to the same block: "
<< sb.era_array_root;
visitor.visit(superblock_invalid(msg.str()));
}

2
era/superblock.h
View File

@@ -59,7 +59,7 @@ namespace era {
era_detail current_detail;
// A btree of undigested era_details
uint64_t bloom_tree_root;
uint64_t writeset_tree_root;
// Big array holding the digested era/block info.
uint64_t era_array_root;

63
era/bloom_tree.cc → era/writeset_tree.cc
View File

@@ -1,10 +1,10 @@
#include "era/bloom_tree.h"
#include "era/writeset_tree.h"
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include "persistent-data/data-structures/bitset.h"
using namespace boost;
using namespace era;
using namespace bloom_tree_detail;
using namespace writeset_tree_detail;
using namespace persistent_data;
using namespace std;
@@ -22,9 +22,9 @@ missing_eras::visit(damage_visitor &v) const {
v.visit(*this);
}
damaged_bloom_filter::damaged_bloom_filter(string const &desc,
uint32_t era,
run<uint32_t> missing_bits)
damaged_writeset::damaged_writeset(string const &desc,
uint32_t era,
run<uint32_t> missing_bits)
: damage(desc),
era_(era),
missing_bits_(missing_bits)
@@ -32,7 +32,7 @@ damaged_bloom_filter::damaged_bloom_filter(string const &desc,
}
void
damaged_bloom_filter::visit(damage_visitor &v) const
damaged_writeset::visit(damage_visitor &v) const
{
v.visit(*this);
}
@@ -40,39 +40,39 @@ damaged_bloom_filter::visit(damage_visitor &v) const
//----------------------------------------------------------------
namespace {
class ll_bloom_visitor : public bitset_detail::bitset_visitor {
class ll_writeset_visitor : public bitset_detail::bitset_visitor {
public:
typedef persistent_data::transaction_manager::ptr tm_ptr;
ll_bloom_visitor(tm_ptr tm,
bloom_tree_detail::bloom_visitor &bloom_v,
bloom_tree_detail::damage_visitor &dv)
ll_writeset_visitor(tm_ptr tm,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv)
: tm_(tm),
bloom_v_(bloom_v),
writeset_v_(writeset_v),
dv_(dv) {
}
void visit(btree_path const &path, era_detail const &era) {
era_ = path[0];
bitset bs(tm_, era.bloom_root, era.nr_bits);
bloom_v_.bloom_begin(era_, era.nr_blocks, era.nr_bits, era.nr_set);
bitset bs(tm_, era.writeset_root, era.nr_bits);
writeset_v_.writeset_begin(era_, era.nr_bits);
bs.walk_bitset(*this);
bloom_v_.bloom_end();
writeset_v_.writeset_end();
}
void visit(uint32_t index, bool value) {
bloom_v_.bit(index, value);
writeset_v_.bit(index, value);
}
void visit(bitset_detail::missing_bits const &d) {
dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_));
dv_.visit(writeset_tree_detail::damaged_writeset("missing bits", era_, d.keys_));
}
private:
tm_ptr tm_;
uint64_t era_;
bloom_tree_detail::bloom_visitor &bloom_v_;
bloom_tree_detail::damage_visitor &dv_;
writeset_tree_detail::writeset_visitor &writeset_v_;
writeset_tree_detail::damage_visitor &dv_;
};
class ll_damage_visitor {
@@ -98,38 +98,37 @@ namespace {
}
void
era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::bloom_visitor &bloom_v,
bloom_tree_detail::damage_visitor &dv)
era::walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv)
{
ll_bloom_visitor ll_bv(tm, bloom_v, dv);
ll_writeset_visitor ll_bv(tm, writeset_v, dv);
ll_damage_visitor ll_dv(dv);
btree_visit_values(tree, ll_bv, ll_dv);
}
namespace {
class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor {
class noop_writeset_visitor : public writeset_tree_detail::writeset_visitor {
public:
void bloom_begin(uint32_t era, uint32_t nr_blocks,
uint32_t nr_bits, uint32_t nr_set) {
void writeset_begin(uint32_t era, uint32_t nr_bits) {
}
void bit(uint32_t index, bool value) {
}
void bloom_end() {
void writeset_end() {
}
};
};
void
era::check_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::damage_visitor &dv)
era::check_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::damage_visitor &dv)
{
noop_bloom_visitor bv;
walk_bloom_tree(tm, tree, bv, dv);
noop_writeset_visitor bv;
walk_writeset_tree(tm, tree, bv, dv);
}
//----------------------------------------------------------------

43
era/bloom_tree.h → era/writeset_tree.h
View File

@@ -1,5 +1,5 @@
#ifndef ERA_BLOOM_TREE_H
#define ERA_BLOOM_TREE_H
#ifndef ERA_WRITESET_TREE_H
#define ERA_WRITESET_TREE_H
#include "era/era_detail.h"
#include "persistent-data/data-structures/btree.h"
@@ -7,7 +7,7 @@
//----------------------------------------------------------------
namespace era {
namespace bloom_tree_detail {
namespace writeset_tree_detail {
class damage_visitor;
class damage {
@@ -34,10 +34,10 @@ namespace era {
run<uint32_t> eras_;
};
struct damaged_bloom_filter : public damage {
damaged_bloom_filter(std::string const &desc,
uint32_t era,
run<uint32_t> missing_bits);
struct damaged_writeset : public damage {
damaged_writeset(std::string const &desc,
uint32_t era,
run<uint32_t> missing_bits);
virtual void visit(damage_visitor &v) const;
uint32_t era_;
@@ -55,32 +55,31 @@ namespace era {
}
virtual void visit(missing_eras const &d) = 0;
virtual void visit(damaged_bloom_filter const &d) = 0;
virtual void visit(damaged_writeset const &d) = 0;
};
class bloom_visitor {
class writeset_visitor {
public:
typedef boost::shared_ptr<bloom_visitor> ptr;
typedef boost::shared_ptr<writeset_visitor> ptr;
virtual ~bloom_visitor() {}
virtual ~writeset_visitor() {}
virtual void bloom_begin(uint32_t era, uint32_t nr_blocks,
uint32_t nr_bits, uint32_t nr_set) = 0;
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) = 0;
virtual void bit(uint32_t index, bool value) = 0;
virtual void bloom_end() = 0;
virtual void writeset_end() = 0;
};
}
typedef persistent_data::btree<1, era_detail_traits> bloom_tree;
typedef persistent_data::btree<1, era_detail_traits> writeset_tree;
void walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::bloom_visitor &bloom_v,
bloom_tree_detail::damage_visitor &dv);
void walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv);
void check_bloom_tree(persistent_data::transaction_manager::ptr tm,
bloom_tree const &tree,
bloom_tree_detail::damage_visitor &dv);
void check_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::damage_visitor &dv);
}
//----------------------------------------------------------------

14
era/xml_format.cc
View File

@@ -33,25 +33,23 @@ namespace {
out_ << "</superblock>" << endl;
}
void begin_bloom(uint32_t era, uint32_t nr_bits,
pd::block_address nr_blocks) {
void begin_writeset(uint32_t era, uint32_t nr_bits) {
indent();
out_ << "<bloom era=\"" << era << "\""
<< " nr_bits=\"" << nr_bits << "\""
<< " nr_blocks=\"" << nr_blocks << "\">" << endl;
out_ << "<writeset era=\"" << era << "\""
<< " nr_bits=\"" << nr_bits << "\">" << endl;
inc();
}
void bloom_bit(uint32_t bit, bool value) {
void writeset_bit(uint32_t bit, bool value) {
indent();
// FIXME: collect all the bits, then uuencode
out_ << "<bit bit=\"" << bit << "\" value=\"" << value << "\">" << endl;
}
void end_bloom() {
void end_writeset() {
dec();
indent();
out_ << "</bloom>" << endl;
out_ << "</writeset>" << endl;
}
void begin_era_array() {