From bfb540cc5b78e48a82b15f1d6f05c3ee44c96f04 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 9 Jan 2014 22:40:34 +0000 Subject: [PATCH] [era] era_superblock_t and era_check --- .gitignore | 2 + Makefile.in | 34 ++++ era/era_check.cc | 249 +++++++++++++++++++++++++ era/superblock.cc | 165 +++++++++++++++- era/superblock.h | 52 +++++- features/era_check.feature | 83 +++++++++ features/step_definitions/era_steps.rb | 16 ++ unit-tests/Makefile.in | 1 + unit-tests/era_superblock_t.cc | 112 +++++++++++ 9 files changed, 709 insertions(+), 5 deletions(-) create mode 100644 era/era_check.cc create mode 100644 features/era_check.feature create mode 100644 features/step_definitions/era_steps.rb create mode 100644 unit-tests/era_superblock_t.cc diff --git a/.gitignore b/.gitignore index 55164d5..7d541d7 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ cache_dump cache_restore cache_repair +era_check + *.metadata bad-metadata Makefile diff --git a/Makefile.in b/Makefile.in index 423025e..82c785f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -25,6 +25,8 @@ PROGRAMS=\ cache_restore \ cache_repair \ \ + era_check \ + \ thin_check \ thin_dump \ thin_restore \ @@ -289,6 +291,38 @@ cache_restore: $(CACHE_RESTORE_OBJECTS) caching/cache_restore.o @echo " [LD] $@" $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT) +#---------------------------------------------------------------- +# Era tools + +ERA_CHECK_SOURCE=\ + base/base64.cc \ + base/error_state.cc \ + base/endian_utils.cc \ + \ + era/superblock.cc \ + era/era_detail.cc \ + \ + persistent-data/checksum.cc \ + persistent-data/error_set.cc \ + persistent-data/file_utils.cc \ + persistent-data/hex_dump.cc \ + persistent-data/lock_tracker.cc \ + persistent-data/data-structures/btree.cc \ + persistent-data/data-structures/bitset.cc \ + persistent-data/space_map.cc \ + persistent-data/space-maps/disk.cc \ + persistent-data/space-maps/recursive.cc \ + persistent-data/space-maps/careful_alloc.cc \ + persistent-data/transaction_manager.cc \ + +ERA_CHECK_OBJECTS=$(subst .cc,.o,$(ERA_CHECK_SOURCE)) + +era_check: $(ERA_CHECK_OBJECTS) era/era_check.o + @echo " [LD] $@" + $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT) + +#---------------------------------------------------------------- + DEPEND_FILES=\ $(subst .cc,.d,$(SOURCE)) \ $(subst .cc,.d,$(TEST_SOURCE)) \ diff --git a/era/era_check.cc b/era/era_check.cc new file mode 100644 index 0000000..de73527 --- /dev/null +++ b/era/era_check.cc @@ -0,0 +1,249 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base/error_state.h" +#include "base/nested_output.h" +#include "era/superblock.h" +#include "persistent-data/block.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space_map.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/transaction_manager.h" +#include "version.h" + +using namespace base; +using namespace boost; +using namespace era; +using namespace persistent_data; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + class reporter_base { + public: + reporter_base(nested_output &o) + : out_(o), + err_(NO_ERROR) { + } + + virtual ~reporter_base() {} + + nested_output &out() { + return out_; + } + + nested_output::nest push() { + return out_.push(); + } + + base::error_state get_error() const { + return err_; + } + + void mplus_error(error_state err) { + err_ = combine_errors(err_, err); + } + + private: + nested_output &out_; + error_state err_; + }; + + class superblock_reporter : public superblock_damage::damage_visitor, reporter_base { + public: + superblock_reporter(nested_output &o) + : reporter_base(o) { + } + + virtual void visit(superblock_damage::superblock_corrupt const &d) { + out() << "superblock is corrupt" << end_message(); + { + nested_output::nest _ = push(); + out() << d.get_desc() << end_message(); + } + + mplus_error(FATAL); + } + + virtual void visit(superblock_damage::superblock_invalid const &d) { + out() << "superblock is invalid" << end_message(); + { + nested_output::nest _ = push(); + out() << d.get_desc() << end_message(); + } + + mplus_error(FATAL); + } + + using reporter_base::get_error; + }; + + //-------------------------------- + + transaction_manager::ptr open_tm(block_manager<>::ptr bm) { + space_map::ptr sm(new core_map(bm->get_nr_blocks())); + sm->inc(SUPERBLOCK_LOCATION); + transaction_manager::ptr tm(new transaction_manager(bm, sm)); + return tm; + } + + //-------------------------------- + + struct flags { + flags() + : superblock_only_(false), + quiet_(false) { + } + + bool superblock_only_; + bool quiet_; + }; + + struct stat guarded_stat(string const &path) { + struct stat info; + + int r = ::stat(path.c_str(), &info); + if (r) { + ostringstream msg; + char buffer[128], *ptr; + + ptr = ::strerror_r(errno, buffer, sizeof(buffer)); + msg << path << ": " << ptr; + throw runtime_error(msg.str()); + } + + return info; + } + + error_state metadata_check(block_manager<>::ptr bm, flags const &fs) { + nested_output out(cerr, 2); + if (fs.quiet_) + out.disable(); + + superblock_reporter sb_rep(out); + + out << "examining superblock" << end_message(); + { + nested_output::nest _ = out.push(); + check_superblock(bm, bm->get_nr_blocks(), sb_rep); + } + + if (sb_rep.get_error() == FATAL) + return FATAL; + +#if 0 + superblock sb = read_superblock(bm); + transaction_manager::ptr tm = open_tm(bm); + + // FIXME: make an error class that's an instance of mplus + return combine_errors(sb_rep.get_error(), + combine_errors(mapping_rep.get_error(), + combine_errors(hint_rep.get_error(), + discard_rep.get_error()))); +#endif + + return FATAL; + } + + int check(string const &path, flags const &fs) { + error_state err; + struct stat info = guarded_stat(path); + + if (!S_ISREG(info.st_mode) && !S_ISBLK(info.st_mode)) { + ostringstream msg; + msg << path << ": " << "Not a block device or regular file"; + throw runtime_error(msg.str()); + } + + block_manager<>::ptr bm = open_bm(path, block_io<>::READ_ONLY); + err = metadata_check(bm, fs); + + return err == NO_ERROR ? 0 : 1; + } + + int check_with_exception_handling(string const &path, flags const &fs) { + int r; + try { + r = check(path, fs); + + } catch (std::exception &e) { + if (!fs.quiet_) + cerr << e.what() << endl; + r = 1; + } + + return r; + + } + + void usage(ostream &out, string const &cmd) { + out << "Usage: " << cmd << " [options] {device|file}" << endl + << "Options:" << endl + << " {-q|--quiet}" << endl + << " {-h|--help}" << endl + << " {-V|--version}" << endl + << " {--super-block-only}" << endl; + } +} + +//---------------------------------------------------------------- + +int main(int argc, char **argv) +{ + int c; + flags fs; + const char shortopts[] = "qhV"; + const struct option longopts[] = { + { "quiet", no_argument, NULL, 'q' }, + { "super-block-only", no_argument, NULL, 1 }, + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, 'V' }, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 1: + fs.superblock_only_ = true; + break; + + case 'h': + usage(cout, basename(argv[0])); + return 0; + + case 'q': + fs.quiet_ = true; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr, basename(argv[0])); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr, basename(argv[0])); + return 1; + } + + return check_with_exception_handling(argv[optind], fs); +} + +//---------------------------------------------------------------- diff --git a/era/superblock.cc b/era/superblock.cc index 561d469..99e51e0 100644 --- a/era/superblock.cc +++ b/era/superblock.cc @@ -5,6 +5,7 @@ using namespace base; using namespace era; +using namespace superblock_damage; using namespace persistent_data; //---------------------------------------------------------------- @@ -53,6 +54,58 @@ namespace { //---------------------------------------------------------------- +superblock_flags::superblock_flags() + : unhandled_flags_(0) +{ +} + +superblock_flags::superblock_flags(uint32_t bits) +{ + if (bits & (1 << CLEAN_SHUTDOWN_BIT)) { + flags_.insert(CLEAN_SHUTDOWN); + bits &= ~(1 << CLEAN_SHUTDOWN_BIT); + } + + unhandled_flags_ = bits; +} + +void +superblock_flags::set_flag(flag f) +{ + flags_.insert(f); +} + +void +superblock_flags::clear_flag(flag f) +{ + flags_.erase(f); +} + +bool +superblock_flags::get_flag(flag f) const +{ + return flags_.find(f) != flags_.end(); +} + +uint32_t +superblock_flags::encode() const +{ + uint32_t r = 0; + + if (get_flag(CLEAN_SHUTDOWN)) + r = r | (1 << CLEAN_SHUTDOWN_BIT); + + return r; +} + +uint32_t +superblock_flags::get_unhandled_flags() const +{ + return unhandled_flags_; +} + +//---------------------------------------------------------------- + superblock::superblock() : csum(0), blocknr(0), @@ -112,9 +165,33 @@ superblock_traits::pack(value_type const &value, disk_type &disk) disk.era_array_root = to_disk(value.era_array_root); } +//-------------------------------- + +superblock_corrupt::superblock_corrupt(std::string const &desc) + : damage(desc) +{ +} + +void +superblock_corrupt::visit(damage_visitor &v) const +{ + v.visit(*this); +} + +superblock_invalid::superblock_invalid(std::string const &desc) + : damage(desc) +{ +} + +void +superblock_invalid::visit(damage_visitor &v) const +{ + v.visit(*this); +} + //---------------------------------------------------------------- -namespace validator { +namespace era_validator { using namespace persistent_data; uint32_t const VERSION = 1; @@ -150,7 +227,7 @@ superblock era::read_superblock(block_manager<>::ptr bm, block_address location) { superblock sb; - block_manager<>::read_ref r = bm->read_lock(location, validator::mk_v()); + block_manager<>::read_ref r = bm->read_lock(location, era_validator::mk_v()); superblock_disk const *sbd = reinterpret_cast(&r.data()); superblock_traits::unpack(*sbd, sb); @@ -160,8 +237,90 @@ era::read_superblock(block_manager<>::ptr bm, block_address location) void era::write_superblock(block_manager<>::ptr bm, superblock const &sb, block_address location) { - block_manager<>::write_ref w = bm->superblock_zero(location, validator::mk_v()); + block_manager<>::write_ref w = bm->superblock_zero(location, era_validator::mk_v()); superblock_traits::pack(sb, *reinterpret_cast(w.data().raw())); } +void +era::check_superblock(superblock const &sb, + block_address nr_metadata_blocks, + damage_visitor &visitor) +{ + if (sb.flags.get_unhandled_flags()) { + ostringstream msg; + msg << "invalid flags: " << sb.flags.get_unhandled_flags(); + visitor.visit(superblock_invalid(msg.str())); + } + + if (sb.blocknr >= nr_metadata_blocks) { + ostringstream msg; + msg << "blocknr out of bounds: " << sb.blocknr << " >= " << nr_metadata_blocks; + visitor.visit(superblock_invalid(msg.str())); + } + + if (sb.magic != SUPERBLOCK_MAGIC) { + ostringstream msg; + msg << "magic in incorrect: " << sb.magic; + visitor.visit(superblock_invalid(msg.str())); + } + + if (sb.version >= VERSION_END) { + ostringstream msg; + msg << "version incorrect: " << sb.version; + visitor.visit(superblock_invalid(msg.str())); + } + + if (sb.version < VERSION_BEGIN) { + ostringstream msg; + msg << "version incorrect: " << sb.version; + visitor.visit(superblock_invalid(msg.str())); + } + + if (sb.metadata_block_size != 8) { + ostringstream msg; + msg << "metadata block size incorrect: " << sb.metadata_block_size; + visitor.visit(superblock_invalid(msg.str())); + } + + if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) { + string msg("bloom tree root points back to the superblock"); + visitor.visit(superblock_invalid(msg)); + } + + if (sb.era_array_root == SUPERBLOCK_LOCATION) { + string msg("era array root points back to the superblock"); + visitor.visit(superblock_invalid(msg)); + } + + if (sb.bloom_tree_root == sb.era_array_root) { + ostringstream msg; + msg << "bloom tree root and era array both point to the same block: " + << sb.era_array_root; + visitor.visit(superblock_invalid(msg.str())); + } +} + +void +era::check_superblock(persistent_data::block_manager<>::ptr bm, + block_address nr_metadata_blocks, + damage_visitor &visitor) +{ + superblock sb; + + try { + sb = read_superblock(bm, SUPERBLOCK_LOCATION); + + } catch (std::exception const &e) { + + // FIXME: what if it fails due to a zero length file? Not + // really a corruption, so much as an io error. Should we + // separate these? + + visitor.visit(superblock_corrupt(e.what())); + } + + check_superblock(sb, nr_metadata_blocks, visitor); +} + + //---------------------------------------------------------------- diff --git a/era/superblock.h b/era/superblock.h index 6d43e12..2ff1179 100644 --- a/era/superblock.h +++ b/era/superblock.h @@ -67,16 +67,64 @@ namespace era { //-------------------------------- + namespace superblock_damage { + + class damage_visitor; + + class damage { + public: + damage(std::string const &desc) + : desc_(desc) { + } + + virtual ~damage() {} + virtual void visit(damage_visitor &v) const = 0; + + std::string const &get_desc() const { + return desc_; + } + + private: + std::string desc_; + }; + + struct superblock_corrupt : public damage { + superblock_corrupt(std::string const &desc); + void visit(damage_visitor &v) const; + }; + + struct superblock_invalid : public damage { + superblock_invalid(std::string const &desc); + void visit(damage_visitor &v) const; + }; + + class damage_visitor { + public: + virtual ~damage_visitor() {} + + void visit(damage const &d); + + virtual void visit(superblock_corrupt const &d) = 0; + virtual void visit(superblock_invalid const &d) = 0; + }; + } + + //-------------------------------- + superblock read_superblock(persistent_data::block_manager<>::ptr bm, persistent_data::block_address location = SUPERBLOCK_LOCATION); void write_superblock(persistent_data::block_manager<>::ptr bm, superblock const &sb, persistent_data::block_address location = SUPERBLOCK_LOCATION); -#if 0 + void check_superblock(superblock const &sb, + persistent_data::block_address nr_metadata_blocks, + superblock_damage::damage_visitor &visitor); + + void check_superblock(persistent_data::block_manager<>::ptr bm, + persistent_data::block_address nr_metadata_blocks, superblock_damage::damage_visitor &visitor); -#endif } //---------------------------------------------------------------- diff --git a/features/era_check.feature b/features/era_check.feature new file mode 100644 index 0000000..430c702 --- /dev/null +++ b/features/era_check.feature @@ -0,0 +1,83 @@ +Feature: era_check + Scenario: print version (-V flag) + When I run `era_check -V` + + Then it should pass with version + + Scenario: print version (--version flag) + When I run `era_check --version` + + Then it should pass with version + + Scenario: print help + When I run `era_check --help` + + Then it should pass + And era_usage to stdout + + Scenario: print help + When I run `era_check -h` + + Then it should pass + And era_usage to stdout + + Scenario: Metadata file must be specified + When I run `era_check` + + Then it should fail + And era_usage to stderr + And the stderr should contain: + + """ + No input file provided. + """ + + Scenario: Metadata file doesn't exist + When I run `era_check /arbitrary/filename` + + Then it should fail + And the stderr should contain: + """ + /arbitrary/filename: No such file or directory + """ + + Scenario: Metadata file cannot be a directory + Given a directory called foo + + When I run `era_check foo` + + Then it should fail + And the stderr should contain: + """ + foo: Not a block device or regular file + """ + + # This test will fail if you're running as root + Scenario: Metadata file exists, but can't be opened + Given input without read permissions + When I run `era_check input` + Then it should fail + And the stderr should contain: + """ + Permission denied + """ + + Scenario: Metadata file full of zeroes + Given input file + And block 1 is zeroed + When I run `era_check input` + Then it should fail + + Scenario: --quiet is observed + Given input file + And block 1 is zeroed + When I run `era_check --quiet input` + Then it should fail + And it should give no output + + Scenario: -q is observed + Given input file + And block 1 is zeroed + When I run `era_check -q input` + Then it should fail + And it should give no output diff --git a/features/step_definitions/era_steps.rb b/features/step_definitions/era_steps.rb new file mode 100644 index 0000000..a1e9f29 --- /dev/null +++ b/features/step_definitions/era_steps.rb @@ -0,0 +1,16 @@ +ERA_USAGE =<(_))).Times(1); + check(); + } + + damage_visitor_mock visitor_; + superblock sb_; + }; +} + +//---------------------------------------------------------------- + +TEST_F(EraSuperblockTests, default_constructed_superblock_is_valid) +{ + check(); +} + +TEST_F(EraSuperblockTests, clean_shutdown_flag_is_valid) +{ + sb_.flags.set_flag(superblock_flags::CLEAN_SHUTDOWN); + check(); +} + +TEST_F(EraSuperblockTests, unhandled_flags_get_set_correctly_and_is_invalid) +{ + uint32_t bad_flag = 1 << 12; + sb_.flags = superblock_flags(bad_flag | 1); + ASSERT_THAT(sb_.flags.get_unhandled_flags(), Eq(bad_flag)); + check_invalid(); +} + +TEST_F(EraSuperblockTests, blocknr_is_in_range) +{ + sb_.blocknr = NR_METADATA_BLOCKS; + check_invalid(); +} + +TEST_F(EraSuperblockTests, magic_is_checked) +{ + sb_.magic = 12345; + check_invalid(); +} + +TEST_F(EraSuperblockTests, version_gt_1_is_checked) +{ + sb_.version = 2; + check_invalid(); +} + +TEST_F(EraSuperblockTests, version_lt_1_is_checked) +{ + sb_.version = 0; + check_invalid(); +} + +TEST_F(EraSuperblockTests, metadata_block_size_checked) +{ + sb_.metadata_block_size = 16; + check_invalid(); +} + +TEST_F(EraSuperblockTests, bloom_tree_root_isnt_0) +{ + sb_.bloom_tree_root = 0; + check_invalid(); +} + +TEST_F(EraSuperblockTests, era_array_root_isnt_0) +{ + sb_.era_array_root = 0; + check_invalid(); +} + +TEST_F(EraSuperblockTests, bloom_root_isnt_era_array_root) +{ + sb_.bloom_tree_root = 10; + sb_.era_array_root = 10; + check_invalid(); +} + +//----------------------------------------------------------------