[era] era_superblock_t and era_check

This commit is contained in:
Joe Thornber 2014-01-09 22:40:34 +00:00
parent 2db5e0265d
commit bfb540cc5b
9 changed files with 709 additions and 5 deletions

2
.gitignore vendored
View File

@ -21,6 +21,8 @@ cache_dump
cache_restore
cache_repair
era_check
*.metadata
bad-metadata
Makefile

View File

@ -25,6 +25,8 @@ PROGRAMS=\
cache_restore \
cache_repair \
\
era_check \
\
thin_check \
thin_dump \
thin_restore \
@ -289,6 +291,38 @@ cache_restore: $(CACHE_RESTORE_OBJECTS) caching/cache_restore.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
#----------------------------------------------------------------
# Era tools
ERA_CHECK_SOURCE=\
base/base64.cc \
base/error_state.cc \
base/endian_utils.cc \
\
era/superblock.cc \
era/era_detail.cc \
\
persistent-data/checksum.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
persistent-data/data-structures/bitset.cc \
persistent-data/space_map.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
ERA_CHECK_OBJECTS=$(subst .cc,.o,$(ERA_CHECK_SOURCE))
era_check: $(ERA_CHECK_OBJECTS) era/era_check.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
#----------------------------------------------------------------
DEPEND_FILES=\
$(subst .cc,.d,$(SOURCE)) \
$(subst .cc,.d,$(TEST_SOURCE)) \

249
era/era_check.cc Normal file
View File

@ -0,0 +1,249 @@
#include <iostream>
#include <sstream>
#include <string>
#include <stdexcept>
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
#include <libgen.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "base/error_state.h"
#include "base/nested_output.h"
#include "era/superblock.h"
#include "persistent-data/block.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space_map.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/transaction_manager.h"
#include "version.h"
using namespace base;
using namespace boost;
using namespace era;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
class reporter_base {
public:
reporter_base(nested_output &o)
: out_(o),
err_(NO_ERROR) {
}
virtual ~reporter_base() {}
nested_output &out() {
return out_;
}
nested_output::nest push() {
return out_.push();
}
base::error_state get_error() const {
return err_;
}
void mplus_error(error_state err) {
err_ = combine_errors(err_, err);
}
private:
nested_output &out_;
error_state err_;
};
class superblock_reporter : public superblock_damage::damage_visitor, reporter_base {
public:
superblock_reporter(nested_output &o)
: reporter_base(o) {
}
virtual void visit(superblock_damage::superblock_corrupt const &d) {
out() << "superblock is corrupt" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
}
mplus_error(FATAL);
}
virtual void visit(superblock_damage::superblock_invalid const &d) {
out() << "superblock is invalid" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
}
mplus_error(FATAL);
}
using reporter_base::get_error;
};
//--------------------------------
transaction_manager::ptr open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}
//--------------------------------
struct flags {
flags()
: superblock_only_(false),
quiet_(false) {
}
bool superblock_only_;
bool quiet_;
};
struct stat guarded_stat(string const &path) {
struct stat info;
int r = ::stat(path.c_str(), &info);
if (r) {
ostringstream msg;
char buffer[128], *ptr;
ptr = ::strerror_r(errno, buffer, sizeof(buffer));
msg << path << ": " << ptr;
throw runtime_error(msg.str());
}
return info;
}
error_state metadata_check(block_manager<>::ptr bm, flags const &fs) {
nested_output out(cerr, 2);
if (fs.quiet_)
out.disable();
superblock_reporter sb_rep(out);
out << "examining superblock" << end_message();
{
nested_output::nest _ = out.push();
check_superblock(bm, bm->get_nr_blocks(), sb_rep);
}
if (sb_rep.get_error() == FATAL)
return FATAL;
#if 0
superblock sb = read_superblock(bm);
transaction_manager::ptr tm = open_tm(bm);
// FIXME: make an error class that's an instance of mplus
return combine_errors(sb_rep.get_error(),
combine_errors(mapping_rep.get_error(),
combine_errors(hint_rep.get_error(),
discard_rep.get_error())));
#endif
return FATAL;
}
int check(string const &path, flags const &fs) {
error_state err;
struct stat info = guarded_stat(path);
if (!S_ISREG(info.st_mode) && !S_ISBLK(info.st_mode)) {
ostringstream msg;
msg << path << ": " << "Not a block device or regular file";
throw runtime_error(msg.str());
}
block_manager<>::ptr bm = open_bm(path, block_io<>::READ_ONLY);
err = metadata_check(bm, fs);
return err == NO_ERROR ? 0 : 1;
}
int check_with_exception_handling(string const &path, flags const &fs) {
int r;
try {
r = check(path, fs);
} catch (std::exception &e) {
if (!fs.quiet_)
cerr << e.what() << endl;
r = 1;
}
return r;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-q|--quiet}" << endl
<< " {-h|--help}" << endl
<< " {-V|--version}" << endl
<< " {--super-block-only}" << endl;
}
}
//----------------------------------------------------------------
int main(int argc, char **argv)
{
int c;
flags fs;
const char shortopts[] = "qhV";
const struct option longopts[] = {
{ "quiet", no_argument, NULL, 'q' },
{ "super-block-only", no_argument, NULL, 1 },
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 1:
fs.superblock_only_ = true;
break;
case 'h':
usage(cout, basename(argv[0]));
return 0;
case 'q':
fs.quiet_ = true;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, basename(argv[0]));
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
return check_with_exception_handling(argv[optind], fs);
}
//----------------------------------------------------------------

View File

@ -5,6 +5,7 @@
using namespace base;
using namespace era;
using namespace superblock_damage;
using namespace persistent_data;
//----------------------------------------------------------------
@ -53,6 +54,58 @@ namespace {
//----------------------------------------------------------------
superblock_flags::superblock_flags()
: unhandled_flags_(0)
{
}
superblock_flags::superblock_flags(uint32_t bits)
{
if (bits & (1 << CLEAN_SHUTDOWN_BIT)) {
flags_.insert(CLEAN_SHUTDOWN);
bits &= ~(1 << CLEAN_SHUTDOWN_BIT);
}
unhandled_flags_ = bits;
}
void
superblock_flags::set_flag(flag f)
{
flags_.insert(f);
}
void
superblock_flags::clear_flag(flag f)
{
flags_.erase(f);
}
bool
superblock_flags::get_flag(flag f) const
{
return flags_.find(f) != flags_.end();
}
uint32_t
superblock_flags::encode() const
{
uint32_t r = 0;
if (get_flag(CLEAN_SHUTDOWN))
r = r | (1 << CLEAN_SHUTDOWN_BIT);
return r;
}
uint32_t
superblock_flags::get_unhandled_flags() const
{
return unhandled_flags_;
}
//----------------------------------------------------------------
superblock::superblock()
: csum(0),
blocknr(0),
@ -112,9 +165,33 @@ superblock_traits::pack(value_type const &value, disk_type &disk)
disk.era_array_root = to_disk<le64>(value.era_array_root);
}
//--------------------------------
superblock_corrupt::superblock_corrupt(std::string const &desc)
: damage(desc)
{
}
void
superblock_corrupt::visit(damage_visitor &v) const
{
v.visit(*this);
}
superblock_invalid::superblock_invalid(std::string const &desc)
: damage(desc)
{
}
void
superblock_invalid::visit(damage_visitor &v) const
{
v.visit(*this);
}
//----------------------------------------------------------------
namespace validator {
namespace era_validator {
using namespace persistent_data;
uint32_t const VERSION = 1;
@ -150,7 +227,7 @@ superblock
era::read_superblock(block_manager<>::ptr bm, block_address location)
{
superblock sb;
block_manager<>::read_ref r = bm->read_lock(location, validator::mk_v());
block_manager<>::read_ref r = bm->read_lock(location, era_validator::mk_v());
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(&r.data());
superblock_traits::unpack(*sbd, sb);
@ -160,8 +237,90 @@ era::read_superblock(block_manager<>::ptr bm, block_address location)
void
era::write_superblock(block_manager<>::ptr bm, superblock const &sb, block_address location)
{
block_manager<>::write_ref w = bm->superblock_zero(location, validator::mk_v());
block_manager<>::write_ref w = bm->superblock_zero(location, era_validator::mk_v());
superblock_traits::pack(sb, *reinterpret_cast<superblock_disk *>(w.data().raw()));
}
void
era::check_superblock(superblock const &sb,
block_address nr_metadata_blocks,
damage_visitor &visitor)
{
if (sb.flags.get_unhandled_flags()) {
ostringstream msg;
msg << "invalid flags: " << sb.flags.get_unhandled_flags();
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.blocknr >= nr_metadata_blocks) {
ostringstream msg;
msg << "blocknr out of bounds: " << sb.blocknr << " >= " << nr_metadata_blocks;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.magic != SUPERBLOCK_MAGIC) {
ostringstream msg;
msg << "magic in incorrect: " << sb.magic;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.version >= VERSION_END) {
ostringstream msg;
msg << "version incorrect: " << sb.version;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.version < VERSION_BEGIN) {
ostringstream msg;
msg << "version incorrect: " << sb.version;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.metadata_block_size != 8) {
ostringstream msg;
msg << "metadata block size incorrect: " << sb.metadata_block_size;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) {
string msg("bloom tree root points back to the superblock");
visitor.visit(superblock_invalid(msg));
}
if (sb.era_array_root == SUPERBLOCK_LOCATION) {
string msg("era array root points back to the superblock");
visitor.visit(superblock_invalid(msg));
}
if (sb.bloom_tree_root == sb.era_array_root) {
ostringstream msg;
msg << "bloom tree root and era array both point to the same block: "
<< sb.era_array_root;
visitor.visit(superblock_invalid(msg.str()));
}
}
void
era::check_superblock(persistent_data::block_manager<>::ptr bm,
block_address nr_metadata_blocks,
damage_visitor &visitor)
{
superblock sb;
try {
sb = read_superblock(bm, SUPERBLOCK_LOCATION);
} catch (std::exception const &e) {
// FIXME: what if it fails due to a zero length file? Not
// really a corruption, so much as an io error. Should we
// separate these?
visitor.visit(superblock_corrupt(e.what()));
}
check_superblock(sb, nr_metadata_blocks, visitor);
}
//----------------------------------------------------------------

View File

@ -67,16 +67,64 @@ namespace era {
//--------------------------------
namespace superblock_damage {
class damage_visitor;
class damage {
public:
damage(std::string const &desc)
: desc_(desc) {
}
virtual ~damage() {}
virtual void visit(damage_visitor &v) const = 0;
std::string const &get_desc() const {
return desc_;
}
private:
std::string desc_;
};
struct superblock_corrupt : public damage {
superblock_corrupt(std::string const &desc);
void visit(damage_visitor &v) const;
};
struct superblock_invalid : public damage {
superblock_invalid(std::string const &desc);
void visit(damage_visitor &v) const;
};
class damage_visitor {
public:
virtual ~damage_visitor() {}
void visit(damage const &d);
virtual void visit(superblock_corrupt const &d) = 0;
virtual void visit(superblock_invalid const &d) = 0;
};
}
//--------------------------------
superblock read_superblock(persistent_data::block_manager<>::ptr bm,
persistent_data::block_address location = SUPERBLOCK_LOCATION);
void write_superblock(persistent_data::block_manager<>::ptr bm,
superblock const &sb,
persistent_data::block_address location = SUPERBLOCK_LOCATION);
#if 0
void check_superblock(superblock const &sb,
persistent_data::block_address nr_metadata_blocks,
superblock_damage::damage_visitor &visitor);
void check_superblock(persistent_data::block_manager<>::ptr bm,
persistent_data::block_address nr_metadata_blocks,
superblock_damage::damage_visitor &visitor);
#endif
}
//----------------------------------------------------------------

View File

@ -0,0 +1,83 @@
Feature: era_check
Scenario: print version (-V flag)
When I run `era_check -V`
Then it should pass with version
Scenario: print version (--version flag)
When I run `era_check --version`
Then it should pass with version
Scenario: print help
When I run `era_check --help`
Then it should pass
And era_usage to stdout
Scenario: print help
When I run `era_check -h`
Then it should pass
And era_usage to stdout
Scenario: Metadata file must be specified
When I run `era_check`
Then it should fail
And era_usage to stderr
And the stderr should contain:
"""
No input file provided.
"""
Scenario: Metadata file doesn't exist
When I run `era_check /arbitrary/filename`
Then it should fail
And the stderr should contain:
"""
/arbitrary/filename: No such file or directory
"""
Scenario: Metadata file cannot be a directory
Given a directory called foo
When I run `era_check foo`
Then it should fail
And the stderr should contain:
"""
foo: Not a block device or regular file
"""
# This test will fail if you're running as root
Scenario: Metadata file exists, but can't be opened
Given input without read permissions
When I run `era_check input`
Then it should fail
And the stderr should contain:
"""
Permission denied
"""
Scenario: Metadata file full of zeroes
Given input file
And block 1 is zeroed
When I run `era_check input`
Then it should fail
Scenario: --quiet is observed
Given input file
And block 1 is zeroed
When I run `era_check --quiet input`
Then it should fail
And it should give no output
Scenario: -q is observed
Given input file
And block 1 is zeroed
When I run `era_check -q input`
Then it should fail
And it should give no output

View File

@ -0,0 +1,16 @@
ERA_USAGE =<<EOF
Usage: era_check [options] {device|file}
Options:
{-q|--quiet}
{-h|--help}
{-V|--version}
{--super-block-only}
EOF
Then /^era_usage to stdout$/ do
assert_partial_output(ERA_USAGE, all_stdout)
end
Then /^era_usage to stderr$/ do
assert_partial_output(ERA_USAGE, all_stderr)
end

View File

@ -59,6 +59,7 @@ TEST_SOURCE=\
unit-tests/cache_superblock_t.cc \
unit-tests/damage_tracker_t.cc \
unit-tests/endian_t.cc \
unit-tests/era_superblock_t.cc \
unit-tests/rmap_visitor_t.cc \
unit-tests/run_set_t.cc \
unit-tests/space_map_t.cc \

View File

@ -0,0 +1,112 @@
#include "gmock/gmock.h"
#include "base/bits.h"
#include "era/superblock.h"
using namespace base;
using namespace era;
using namespace superblock_damage;
using namespace testing;
//----------------------------------------------------------------
namespace {
unsigned const NR_METADATA_BLOCKS = 100;
class damage_visitor_mock : public damage_visitor {
public:
MOCK_METHOD1(visit, void (superblock_corrupt const &));
MOCK_METHOD1(visit, void (superblock_invalid const &));
};
class EraSuperblockTests : public Test {
public:
EraSuperblockTests() {
sb_.bloom_tree_root = 1;
sb_.era_array_root = 2;
}
void check() {
check_superblock(sb_, NR_METADATA_BLOCKS, visitor_);
}
void check_invalid() {
EXPECT_CALL(visitor_, visit(Matcher<superblock_invalid const &>(_))).Times(1);
check();
}
damage_visitor_mock visitor_;
superblock sb_;
};
}
//----------------------------------------------------------------
TEST_F(EraSuperblockTests, default_constructed_superblock_is_valid)
{
check();
}
TEST_F(EraSuperblockTests, clean_shutdown_flag_is_valid)
{
sb_.flags.set_flag(superblock_flags::CLEAN_SHUTDOWN);
check();
}
TEST_F(EraSuperblockTests, unhandled_flags_get_set_correctly_and_is_invalid)
{
uint32_t bad_flag = 1 << 12;
sb_.flags = superblock_flags(bad_flag | 1);
ASSERT_THAT(sb_.flags.get_unhandled_flags(), Eq(bad_flag));
check_invalid();
}
TEST_F(EraSuperblockTests, blocknr_is_in_range)
{
sb_.blocknr = NR_METADATA_BLOCKS;
check_invalid();
}
TEST_F(EraSuperblockTests, magic_is_checked)
{
sb_.magic = 12345;
check_invalid();
}
TEST_F(EraSuperblockTests, version_gt_1_is_checked)
{
sb_.version = 2;
check_invalid();
}
TEST_F(EraSuperblockTests, version_lt_1_is_checked)
{
sb_.version = 0;
check_invalid();
}
TEST_F(EraSuperblockTests, metadata_block_size_checked)
{
sb_.metadata_block_size = 16;
check_invalid();
}
TEST_F(EraSuperblockTests, bloom_tree_root_isnt_0)
{
sb_.bloom_tree_root = 0;
check_invalid();
}
TEST_F(EraSuperblockTests, era_array_root_isnt_0)
{
sb_.era_array_root = 0;
check_invalid();
}
TEST_F(EraSuperblockTests, bloom_root_isnt_era_array_root)
{
sb_.bloom_tree_root = 10;
sb_.era_array_root = 10;
check_invalid();
}
//----------------------------------------------------------------