From 6bff674720c4d8e7cdbba819dfb7fdb9db2a5f0c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 3 Nov 2011 14:44:00 +0000 Subject: [PATCH] thin_restore works (well thin_repair thinks it does). --- Makefile | 8 +- block.h | 6 ++ block.tcc | 23 ++++- btree.h | 18 +--- btree.tcc | 66 ++++++++++++- btree_checker.h | 3 +- metadata.cc | 132 ++++++++++++++++++------- metadata.h | 4 +- metadata_checker.cc | 4 +- metadata_disk_structures.h | 2 + restore_emitter.cc | 6 +- space_map_disk.cc | 78 +++++++++++---- space_map_disk.h | 5 +- thin_restore.cc | 11 ++- transaction_manager.cc | 4 +- unit-tests/space_map_disk_t.cc | 170 +++++++++++++++++++++++++++------ 16 files changed, 423 insertions(+), 117 deletions(-) diff --git a/Makefile b/Makefile index 1cb844e..c18dcdd 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,14 @@ SOURCE=\ transaction_manager.cc \ xml_format.cc +PROGRAM_SOURCE=\ + thin_dump.cc \ + thin_repair.cc \ + thin_restore.cc + OBJECTS=$(subst .cc,.o,$(SOURCE)) TOP_DIR:=$(PWD) -CPPFLAGS=-Wall -g -I$(TOP_DIR) -O8 +CPPFLAGS=-Wall -g -I$(TOP_DIR) #CPPFLAGS=-Wall -std=c++0x -g -I$(TOP_DIR) LIBS=-lstdc++ -lboost_program_options -lexpat @@ -50,3 +55,4 @@ thin_repair: $(OBJECTS) thin_repair.o include unit-tests/Makefile.in include $(subst .cc,.d,$(SOURCE)) include $(subst .cc,.d,$(TEST_SOURCE)) +include $(subst .cc,.d,$(PROGRAM_SOURCE)) diff --git a/block.h b/block.h index e7d3144..c6033a6 100644 --- a/block.h +++ b/block.h @@ -200,6 +200,12 @@ namespace persistent_data { typedef std::map > held_map; mutable held_map held_locks_; }; + + // A little utility to help build validators + inline block_manager<>::validator::ptr + mk_validator(block_manager<>::validator *v) { + return block_manager<>::validator::ptr(v); + } } #include "block.tcc" diff --git a/block.tcc b/block.tcc index 260398b..004b5f4 100644 --- a/block.tcc +++ b/block.tcc @@ -8,6 +8,7 @@ #include #include +#include using namespace boost; using namespace persistent_data; @@ -20,7 +21,8 @@ block_io::block_io(std::string const &path, block_address nr_blocks, : nr_blocks_(nr_blocks), writeable_(writeable) { - fd_ = ::open(path.c_str(), writeable ? (O_RDWR | O_CREAT) : O_RDONLY, 0666); + // fd_ = ::open(path.c_str(), writeable ? (O_RDWR | O_CREAT) : O_RDONLY, 0666); + fd_ = ::open(path.c_str(), writeable ? O_RDWR : O_RDONLY, 0666); if (fd_ < 0) throw std::runtime_error("couldn't open file"); } @@ -75,8 +77,17 @@ block_io::write_buffer(block_address location, const_buffer &buffer) } } while (remaining && ((n > 0) || (n == EINTR) || (n == EAGAIN))); - if (n < 0) - throw std::runtime_error("write failed"); + if (n < 0) { + std::ostringstream out; + out << "write failed to block " << location + << ", block size = " << BlockSize + << ", remaining = " << remaining + << ", n = " << n + << ", errno = " << errno + << ", fd_ = " << fd_ + << std::endl; + throw std::runtime_error(out.str()); + } } //---------------------------------------------------------------- @@ -277,6 +288,7 @@ block_manager::superblock(block_address location, if (cached_block) { (*cached_block)->check_write_lockable(); (*cached_block)->bt_ = BT_SUPERBLOCK; + (*cached_block)->validator_ = v; return write_ref(*this, *cached_block); } @@ -297,10 +309,13 @@ block_manager::superblock_zero(block_address location, if (cached_block) { (*cached_block)->check_write_lockable(); memset(&(*cached_block)->data_, 0, BlockSize); + (*cached_block)->validator_ = v; return write_ref(*this, *cached_block); } - block_ptr b(new block(io_, location, BT_SUPERBLOCK, v, true)); + block_ptr b(new block(io_, location, BT_SUPERBLOCK, + mk_validator(new noop_validator), true)); + b->validator_ = v; cache_.insert(b); return write_ref(*this, b); } diff --git a/btree.h b/btree.h index 4666aab..3ef0705 100644 --- a/btree.h +++ b/btree.h @@ -177,11 +177,7 @@ namespace persistent_data { : tm_(tm) { } - void step(block_address b) { - spine_.push_back(tm_->read_lock(b)); - if (spine_.size() > 2) - spine_.pop_front(); - } + void step(block_address b); template node_ref get_node() { @@ -203,17 +199,7 @@ namespace persistent_data { } // true if the children of the shadow need incrementing - bool step(block_address b) { - pair p = tm_->shadow(b); - try { - step(p.first); - } catch (...) { - tm_->get_sm()->dec(p.first.get_location()); - throw; - } - return p.second; - } - + bool step(block_address b); void step(transaction_manager::write_ref b) { spine_.push_back(b); if (spine_.size() == 1) diff --git a/btree.tcc b/btree.tcc index 93461a0..602fb3b 100644 --- a/btree.tcc +++ b/btree.tcc @@ -1,4 +1,6 @@ #include "btree.h" + +#include "checksum.h" #include "transaction_manager.h" #include @@ -9,6 +11,62 @@ using namespace std; //---------------------------------------------------------------- +namespace { + struct btree_node_validator : public block_manager<>::validator { + virtual void check(block_manager<>::const_buffer &b, block_address location) const { + disk_node const *data = reinterpret_cast(&b); + node_header const *n = &data->header; + crc32c sum(BTREE_CSUM_XOR); + sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(n->csum)) + throw runtime_error("bad checksum in btree node"); + + if (to_cpu(n->blocknr) != location) + throw runtime_error("bad block nr in btree node"); + } + + virtual void prepare(block_manager<>::buffer &b, block_address location) const { + disk_node *data = reinterpret_cast(&b); + node_header *n = &data->header; + n->blocknr = to_disk(location); + + crc32c sum(BTREE_CSUM_XOR); + sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + n->csum = to_disk(sum.get_sum()); + } + }; + + block_manager<>::validator::ptr + btree_validator() { + return block_manager<>::validator::ptr(new btree_node_validator); + } +} + +//---------------------------------------------------------------- + +inline void +ro_spine::step(block_address b) +{ + spine_.push_back(tm_->read_lock(b, btree_validator())); + if (spine_.size() > 2) + spine_.pop_front(); +} + +inline bool +shadow_spine::step(block_address b) +{ + pair p = tm_->shadow(b, btree_validator()); + try { + step(p.first); + } catch (...) { + tm_->get_sm()->dec(p.first.get_location()); + throw; + } + return p.second; +} + +//---------------------------------------------------------------- + template node_ref::node_ref(block_address location, disk_node *raw) : location_(location), @@ -291,7 +349,7 @@ btree(typename transaction_manager::ptr tm, { using namespace btree_detail; - write_ref root = tm_->new_block(); + write_ref root = tm_->new_block(btree_validator()); leaf_node n = to_node(root); n.set_type(btree_detail::LEAF); @@ -511,13 +569,13 @@ split_beneath(btree_detail::shadow_spine &spine, node_type type; unsigned nr_left, nr_right; - write_ref left = tm_->new_block(); + write_ref left = tm_->new_block(btree_validator()); node_ref l = to_node(left); l.set_nr_entries(0); l.set_max_entries(); l.set_value_size(sizeof(typename ValueTraits::disk_type)); - write_ref right = tm_->new_block(); + write_ref right = tm_->new_block(btree_validator()); node_ref r = to_node(right); r.set_nr_entries(0); r.set_max_entries(); @@ -570,7 +628,7 @@ split_sibling(btree_detail::shadow_spine &spine, node_ref l = spine.template get_node(); block_address left = spine.get_block(); - write_ref right = tm_->new_block(); + write_ref right = tm_->new_block(btree_validator()); node_ref r = to_node(right); unsigned nr_left = l.get_nr_entries() / 2; diff --git a/btree_checker.h b/btree_checker.h index 17b1d02..3456130 100644 --- a/btree_checker.h +++ b/btree_checker.h @@ -5,7 +5,6 @@ #include "checksum.h" #include "error_set.h" -#include "hex_dump.h" #include #include @@ -170,7 +169,7 @@ namespace persistent_data { std::ostringstream out; out << "checksum error for block " << n.get_block_nr() << ", sum was " << sum.get_sum() - << ", expected " << n.get_checksum(); + << ", on disk " << n.get_checksum(); errs_->add_child(out.str()); throw runtime_error(out.str()); } diff --git a/metadata.cc b/metadata.cc index 1032f44..9ea5340 100644 --- a/metadata.cc +++ b/metadata.cc @@ -19,8 +19,27 @@ namespace { uint32_t const VERSION = 1; unsigned const METADATA_CACHE_SIZE = 1024; unsigned const SECTOR_TO_BLOCK_SHIFT = 3; + uint32_t const SUPERBLOCK_CSUM_SEED = 160774; - block_address get_nr_blocks(string const &path) { + struct superblock_validator : public block_manager<>::validator { + virtual void check(block_manager<>::const_buffer &b, block_address location) const { + superblock_disk const *sbd = reinterpret_cast(&b); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum_)) + throw runtime_error("bad checksum in superblock"); + } + + virtual void prepare(block_manager<>::buffer &b, block_address location) const { + superblock_disk *sbd = reinterpret_cast(&b); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); + sbd->csum_ = to_disk(sum.get_sum()); + } + }; + + block_address + get_nr_blocks(string const &path) { struct stat info; block_address nr_blocks; @@ -52,55 +71,93 @@ namespace { } transaction_manager::ptr - open_tm(string const &dev_path) { + open_tm(string const &dev_path, bool writeable) { block_address nr_blocks = get_nr_blocks(dev_path); - block_manager<>::ptr bm(new block_manager<>(dev_path, nr_blocks, 8)); + block_manager<>::ptr bm(new block_manager<>(dev_path, nr_blocks, 1, writeable)); space_map::ptr sm(new core_map(nr_blocks)); + sm->inc(SUPERBLOCK_LOCATION); transaction_manager::ptr tm(new transaction_manager(bm, sm)); return tm; } - superblock read_superblock(block_manager<>::ptr bm) { + superblock + read_superblock(block_manager<>::ptr bm) { superblock sb; - block_manager<>::read_ref r = bm->read_lock(SUPERBLOCK_LOCATION); + block_manager<>::read_ref r = bm->read_lock(SUPERBLOCK_LOCATION, + mk_validator(new superblock_validator)); superblock_disk const *sbd = reinterpret_cast(&r.data()); - - crc32c sum(160774); // FIXME: magic number - sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); - if (sum.get_sum() != to_cpu(sbd->csum_)) { - ostringstream out; - out << "bad checksum in superblock, calculated " - << sum.get_sum() - << ", superblock contains " << to_cpu(sbd->csum_); - throw runtime_error(out.str()); - } - superblock_traits::unpack(*sbd, sb); return sb; } + + void + copy_space_maps(space_map::ptr lhs, space_map::ptr rhs) { + for (block_address b = 0; b < rhs->get_nr_blocks(); b++) { + uint32_t count = rhs->get_count(b); + if (count > 0) + lhs->set_count(b, rhs->get_count(b)); + } + } } //---------------------------------------------------------------- -metadata::metadata(std::string const &dev_path, open_type ot) - : tm_(open_tm(dev_path)), - sb_(read_superblock(tm_->get_bm())), - metadata_sm_(open_metadata_sm(tm_, static_cast(&sb_.metadata_space_map_root_))), - data_sm_(open_disk_sm(tm_, static_cast(&sb_.data_space_map_root_))), - details_(new detail_tree(tm_, sb_.device_details_root_, device_details_traits::ref_counter())), - mappings_top_level_(new dev_tree(tm_, sb_.data_mapping_root_, mtree_ref_counter(tm_))), - mappings_(new mapping_tree(tm_, sb_.data_mapping_root_, block_time_ref_counter(data_sm_))) +metadata::metadata(std::string const &dev_path, open_type ot, + sector_t data_block_size, block_address nr_data_blocks) { - tm_->set_sm(open_metadata_sm(tm_, sb_.metadata_space_map_root_)); + switch (ot) { + case OPEN: + tm_ = open_tm(dev_path, false); + sb_ = read_superblock(tm_->get_bm()); + metadata_sm_ = open_metadata_sm(tm_, &sb_.metadata_space_map_root_); + tm_->set_sm(metadata_sm_); + + data_sm_ = open_disk_sm(tm_, static_cast(&sb_.data_space_map_root_)); + details_ = detail_tree::ptr(new detail_tree(tm_, sb_.device_details_root_, device_details_traits::ref_counter())); + mappings_top_level_ = dev_tree::ptr(new dev_tree(tm_, sb_.data_mapping_root_, mtree_ref_counter(tm_))); + mappings_ = mapping_tree::ptr(new mapping_tree(tm_, sb_.data_mapping_root_, block_time_ref_counter(data_sm_))); + break; + + case CREATE: + tm_ = open_tm(dev_path, true); + space_map::ptr core = tm_->get_sm(); + metadata_sm_ = create_metadata_sm(tm_, tm_->get_bm()->get_nr_blocks()); + copy_space_maps(metadata_sm_, core); + tm_->set_sm(metadata_sm_); + + data_sm_ = create_disk_sm(tm_, nr_data_blocks); + details_ = detail_tree::ptr(new detail_tree(tm_, device_details_traits::ref_counter())); + mappings_ = mapping_tree::ptr(new mapping_tree(tm_, block_time_ref_counter(data_sm_))); + mappings_top_level_ = dev_tree::ptr(new dev_tree(tm_, mappings_->get_root(), mtree_ref_counter(tm_))); + + ::memset(&sb_, 0, sizeof(sb_)); + sb_.magic_ = SUPERBLOCK_MAGIC; + sb_.data_mapping_root_ = mappings_->get_root(); + sb_.device_details_root_ = details_->get_root(); + sb_.data_block_size_ = data_block_size; + sb_.metadata_block_size_ = MD_BLOCK_SIZE; + sb_.metadata_nr_blocks_ = tm_->get_bm()->get_nr_blocks(); + + break; + } } -#if 0 - ::memset(&sb_, 0, sizeof(sb_)); - sb_.data_mapping_root_ = mappings_.get_root(); - sb_.device_details_root_ = details_.get_root(); - sb_.metadata_block_size_ = MD_BLOCK_SIZE; - sb_.metadata_nr_blocks_ = tm_->get_bm()->get_nr_blocks(); -#endif +namespace { + void print_superblock(superblock const &sb) { + using namespace std; + + cerr << "superblock " << sb.csum_ << endl + << "flags " << sb.flags_ << endl + << "blocknr " << sb.blocknr_ << endl + << "transaction id " << sb.trans_id_ << endl + << "data mapping root " << sb.data_mapping_root_ << endl + << "details root " << sb.device_details_root_ << endl + << "data block size " << sb.data_block_size_ << endl + << "metadata block size " << sb.metadata_block_size_ << endl + << "metadata nr blocks " << sb.metadata_nr_blocks_ << endl + ; + } +} void metadata::commit() @@ -108,9 +165,16 @@ metadata::commit() sb_.data_mapping_root_ = mappings_->get_root(); sb_.device_details_root_ = details_->get_root(); - // FIXME: commit and copy the space map roots + data_sm_->commit(); + data_sm_->copy_root(&sb_.data_space_map_root_, sizeof(sb_.data_space_map_root_)); - write_ref superblock = tm_->get_bm()->superblock(SUPERBLOCK_LOCATION); + metadata_sm_->commit(); + metadata_sm_->copy_root(&sb_.metadata_space_map_root_, sizeof(sb_.metadata_space_map_root_)); + print_superblock(sb_); + + superblock_validator v; + write_ref superblock = tm_->get_bm()->superblock_zero(SUPERBLOCK_LOCATION, + mk_validator(new superblock_validator())); superblock_disk *disk = reinterpret_cast(superblock.data()); superblock_traits::pack(sb_, *disk); } diff --git a/metadata.h b/metadata.h index eebe006..91dfb1a 100644 --- a/metadata.h +++ b/metadata.h @@ -129,7 +129,9 @@ namespace thin_provisioning { OPEN }; - metadata(std::string const &dev_path, open_type ot); + metadata(std::string const &dev_path, open_type ot, + sector_t data_block_size = 128, + block_address nr_data_blocks = 0); // Only used if CREATE void commit(); diff --git a/metadata_checker.cc b/metadata_checker.cc index a826f5c..892a258 100644 --- a/metadata_checker.cc +++ b/metadata_checker.cc @@ -134,10 +134,11 @@ thin_provisioning::metadata_check(metadata::ptr md) mapping_validator::ptr mv(new mapping_validator(metadata_counter, data_counter)); md->mappings_->visit(mv); - set const &mapped_devs = mv->get_devices(); + set const &mapped_devs = mv->get_devices(); details_validator::ptr dv(new details_validator(metadata_counter)); md->details_->visit(dv); + set const &details_devs = dv->get_devices(); for (set::const_iterator it = mapped_devs.begin(); it != mapped_devs.end(); ++it) @@ -150,6 +151,7 @@ thin_provisioning::metadata_check(metadata::ptr md) metadata_counter.inc(SUPERBLOCK_LOCATION); md->metadata_sm_->check(metadata_counter); + md->data_sm_->check(metadata_counter); errors->add_child(check_ref_counts("Errors in metadata block reference counts", metadata_counter, md->metadata_sm_)); diff --git a/metadata_disk_structures.h b/metadata_disk_structures.h index 5e1efd8..708d79b 100644 --- a/metadata_disk_structures.h +++ b/metadata_disk_structures.h @@ -65,6 +65,7 @@ namespace thin_provisioning { __le64 metadata_nr_blocks_; __le32 compat_flags_; + __le32 compat_ro_flags_; __le32 incompat_flags_; } __attribute__ ((packed)); @@ -97,6 +98,7 @@ namespace thin_provisioning { uint64_t metadata_nr_blocks_; uint32_t compat_flags_; + uint32_t compat_ro_flags_; uint32_t incompat_flags_; }; diff --git a/restore_emitter.cc b/restore_emitter.cc index 5a2a874..b14216f 100644 --- a/restore_emitter.cc +++ b/restore_emitter.cc @@ -38,7 +38,6 @@ namespace { throw runtime_error("missing superblock"); md_->commit(); - in_superblock_ = false; } @@ -64,7 +63,6 @@ namespace { block_time_ref_counter(md_->data_sm_))); md_->mappings_top_level_->insert(key, new_tree->get_root()); md_->mappings_->set_root(md_->mappings_top_level_->get_root()); // FIXME: ugly - current_device_ = optional(dev); } @@ -99,12 +97,14 @@ namespace { bt.time_ = time; md_->mappings_->insert(key, bt); md_->mappings_top_level_->set_root(md_->mappings_->get_root()); + md_->data_sm_->inc(data_block); } private: bool device_exists(thin_dev_t dev) const { uint64_t key[1] = {dev}; - return md_->details_->lookup(key); + detail_tree::maybe_value v = md_->details_->lookup(key); + return v; } metadata::ptr md_; diff --git a/space_map_disk.cc b/space_map_disk.cc index 26a4e9e..a0f4362 100644 --- a/space_map_disk.cc +++ b/space_map_disk.cc @@ -1,5 +1,6 @@ #include "space_map_disk.h" +#include "checksum.h" #include "endian_utils.h" #include "math_utils.h" #include "space_map_disk_structures.h" @@ -14,6 +15,35 @@ using namespace sm_disk_detail; //---------------------------------------------------------------- namespace { + uint64_t const BITMAP_CSUM_XOR = 240779; + + struct bitmap_block_validator : public block_manager<>::validator { + virtual void check(block_manager<>::const_buffer &b, block_address location) const { + bitmap_header const *data = reinterpret_cast(&b); + crc32c sum(BITMAP_CSUM_XOR); + sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(data->csum)) + throw runtime_error("bad checksum in space map bitmap"); + + if (to_cpu(data->blocknr) != location) + throw runtime_error("bad block nr in space map bitmap"); + } + + virtual void prepare(block_manager<>::buffer &b, block_address location) const { + bitmap_header *data = reinterpret_cast(&b); + data->blocknr = to_disk(location); + + crc32c sum(BITMAP_CSUM_XOR); + sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); + data->csum = to_disk(sum.get_sum()); + } + }; + + block_manager<>::validator::ptr + bitmap_validator() { + return block_manager<>::validator::ptr(new bitmap_block_validator()); + } + class bitmap { public: typedef transaction_manager::read_ref read_ref; @@ -64,16 +94,16 @@ namespace { } } - unsigned find_free(unsigned end) { - for (unsigned i = ie_.none_free_before_; i < end; i++) { + boost::optional find_free(unsigned begin, unsigned end) { + for (unsigned i = max(begin, ie_.none_free_before_); i < end; i++) { if (lookup(i) == 0) { insert(i, 1); ie_.none_free_before_ = i + 1; - return i; + return boost::optional(i); } } - throw std::runtime_error("no free entry in bitmap"); + return boost::optional(); } index_entry const &get_ie() const { @@ -213,20 +243,23 @@ namespace { } block_address new_block() { - // silly to always start searching from the + // FIXME: silly to always start searching from the // beginning. block_address nr_indexes = div_up(nr_blocks_, entries_per_block_); for (block_address index = 0; index < nr_indexes; index++) { index_entry ie = find_ie(index); bitmap bm(tm_, ie); - block_address b = bm.find_free((index == nr_indexes - 1) ? - nr_blocks_ % entries_per_block_ : entries_per_block_); - save_ie(b, bm.get_ie()); - nr_allocated_++; - b = (index * entries_per_block_) + b; - assert(get_count(b) == 1); - return b; + optional maybe_b = bm.find_free(0, (index == nr_indexes - 1) ? + nr_blocks_ % entries_per_block_ : entries_per_block_); + if (maybe_b) { + block_address b = *maybe_b; + save_ie(index, bm.get_ie()); + nr_allocated_++; + b = (index * entries_per_block_) + b; + assert(get_count(b) == 1); + return b; + } } throw runtime_error("out of space"); @@ -242,9 +275,9 @@ namespace { block_address bitmap_count = div_up(nr_blocks, entries_per_block_); block_address old_bitmap_count = div_up(nr_blocks_, entries_per_block_); for (block_address i = old_bitmap_count; i < bitmap_count; i++) { - write_ref wr = tm_->new_block(); + write_ref wr = tm_->new_block(bitmap_validator()); - struct index_entry ie; + index_entry ie; ie.blocknr_ = wr.get_location(); ie.nr_free_ = i == (bitmap_count - 1) ? (nr_blocks % entries_per_block_) : entries_per_block_; @@ -324,7 +357,7 @@ namespace { index_entry ie = find_ie(b / entries_per_block_); bitmap bm(tm_, ie); bm.insert(b % entries_per_block_, n); - save_ie(b, bm.get_ie()); + save_ie(b / entries_per_block_, bm.get_ie()); } ref_t lookup_ref_count(block_address b) const { @@ -436,6 +469,7 @@ namespace { v.nr_allocated_ = sm_disk_base::get_nr_allocated(); v.bitmap_root_ = bitmaps_.get_root(); v.ref_count_root_ = sm_disk_base::get_ref_count_root(); + sm_root_traits::pack(v, d); ::memcpy(dest, &d, sizeof(d)); } @@ -477,8 +511,8 @@ namespace { sm_metadata(transaction_manager::ptr tm) : sm_disk_base(tm), + bitmap_root_(tm->get_sm()->new_block()), // FIXME: add bitmap_index validator entries_(MAX_METADATA_BITMAPS) { - // FIXME: allocate a new bitmap root } sm_metadata(transaction_manager::ptr tm, @@ -505,6 +539,7 @@ namespace { v.nr_allocated_ = sm_disk_base::get_nr_allocated(); v.bitmap_root_ = bitmap_root_; v.ref_count_root_ = sm_disk_base::get_ref_count_root(); + sm_root_traits::pack(v, d); ::memcpy(dest, &d, sizeof(d)); } @@ -582,13 +617,22 @@ persistent_data::open_disk_sm(transaction_manager::ptr tm, void *root) } checked_space_map::ptr -persistent_data::open_metadata_sm(transaction_manager::ptr tm, void * root) +persistent_data::create_metadata_sm(transaction_manager::ptr tm, block_address nr_blocks) +{ + checked_space_map::ptr sm(new sm_metadata(tm)); + sm->extend(nr_blocks); + return sm; +} + +checked_space_map::ptr +persistent_data::open_metadata_sm(transaction_manager::ptr tm, void *root) { sm_root_disk d; sm_root v; ::memcpy(&d, root, sizeof(d)); sm_root_traits::unpack(d, v); + return checked_space_map::ptr(new sm_metadata(tm, v)); } diff --git a/space_map_disk.h b/space_map_disk.h index 3038dda..f79788e 100644 --- a/space_map_disk.h +++ b/space_map_disk.h @@ -21,7 +21,10 @@ namespace persistent_data { open_disk_sm(transaction_manager::ptr tm, void *root); checked_space_map::ptr - open_metadata_sm(transaction_manager::ptr tm, void * root); + create_metadata_sm(transaction_manager::ptr tm, block_address nr_blocks); + + checked_space_map::ptr + open_metadata_sm(transaction_manager::ptr tm, void *root); } //---------------------------------------------------------------- diff --git a/thin_restore.cc b/thin_restore.cc index 21250dc..521fde3 100644 --- a/thin_restore.cc +++ b/thin_restore.cc @@ -18,16 +18,21 @@ namespace po = boost::program_options; namespace { void restore(string const &backup_file, string const &dev) { - metadata::ptr md(new metadata(dev, metadata::CREATE)); + // FIXME: hard coded + block_address const NR_BLOCKS = 100000; + + metadata::ptr md(new metadata(dev, metadata::CREATE, 128, NR_BLOCKS)); emitter::ptr restorer = create_restore_emitter(md); ifstream in(backup_file.c_str(), ifstream::in); - try { + // FIXME: + //try { parse_xml(in, restorer); - +#if 0 } catch (...) { in.close(); throw; } +#endif } void usage(po::options_description const &desc) { diff --git a/transaction_manager.cc b/transaction_manager.cc index ed26b6e..4179817 100644 --- a/transaction_manager.cc +++ b/transaction_manager.cc @@ -98,8 +98,8 @@ pair transaction_manager::shadow(block_address orig, validator v) { if (is_shadow(orig) && - sm_->count_possibly_greater_than_one(orig)) - return make_pair(bm_->write_lock(orig), false); + !sm_->count_possibly_greater_than_one(orig)) + return make_pair(bm_->write_lock(orig, v), false); read_ref src = bm_->read_lock(orig, v); write_ref dest = bm_->write_lock_zero(sm_->new_block(), v); diff --git a/unit-tests/space_map_disk_t.cc b/unit-tests/space_map_disk_t.cc index e91b525..e95f14e 100644 --- a/unit-tests/space_map_disk_t.cc +++ b/unit-tests/space_map_disk_t.cc @@ -1,5 +1,5 @@ #include "space_map_disk.h" -#include "core_map.h" +#include "space_map_core.h" #define BOOST_TEST_MODULE SpaceMapDiskTests #include @@ -11,13 +11,14 @@ using namespace persistent_data; //---------------------------------------------------------------- namespace { - block_address const NR_BLOCKS = 10237; + block_address const NR_BLOCKS = 1000; // FIXME: bump up block_address const SUPERBLOCK = 0; + block_address const MAX_LOCKS = 8; transaction_manager::ptr create_tm() { block_manager<>::ptr bm( - new block_manager<>("./test.data", NR_BLOCKS)); + new block_manager<>("./test.data", NR_BLOCKS, MAX_LOCKS, true)); space_map::ptr sm(new core_map(1024)); transaction_manager::ptr tm( new transaction_manager(bm, sm)); @@ -26,27 +27,26 @@ namespace { persistent_space_map::ptr create_sm_disk() { - auto tm = create_tm(); + transaction_manager::ptr tm = create_tm(); return persistent_data::create_disk_sm(tm, NR_BLOCKS); } + + persistent_space_map::ptr + create_sm_metadata() { + transaction_manager::ptr tm = create_tm(); + return persistent_data::create_metadata_sm(tm, NR_BLOCKS); + } } //---------------------------------------------------------------- -BOOST_AUTO_TEST_CASE(reopen_an_sm) +void _test_get_nr_blocks(space_map::ptr sm) { - auto sm = create_sm_disk(); -} - -BOOST_AUTO_TEST_CASE(test_get_nr_blocks) -{ - auto sm = create_sm_disk(); BOOST_CHECK_EQUAL(sm->get_nr_blocks(), NR_BLOCKS); } -BOOST_AUTO_TEST_CASE(test_get_nr_free) +void _test_get_nr_free(space_map::ptr sm) { - auto sm = create_sm_disk(); BOOST_CHECK_EQUAL(sm->get_nr_free(), NR_BLOCKS); for (unsigned i = 0; i < NR_BLOCKS; i++) { @@ -60,18 +60,16 @@ BOOST_AUTO_TEST_CASE(test_get_nr_free) } } -BOOST_AUTO_TEST_CASE(test_throws_no_space) +void _test_throws_no_space(space_map::ptr sm) { - auto sm = create_sm_disk(); for (unsigned i = 0; i < NR_BLOCKS; i++) sm->new_block(); BOOST_CHECK_THROW(sm->new_block(), std::runtime_error); } -BOOST_AUTO_TEST_CASE(test_inc_and_dec) +void _test_inc_and_dec(space_map::ptr sm) { - auto sm = create_sm_disk(); block_address b = 63; for (unsigned i = 0; i < 50; i++) { @@ -85,9 +83,8 @@ BOOST_AUTO_TEST_CASE(test_inc_and_dec) } } -BOOST_AUTO_TEST_CASE(test_not_allocated_twice) +void _test_not_allocated_twice(space_map::ptr sm) { - auto sm = create_sm_disk(); block_address b = sm->new_block(); try { @@ -97,16 +94,14 @@ BOOST_AUTO_TEST_CASE(test_not_allocated_twice) } } -BOOST_AUTO_TEST_CASE(test_set_count) +void _test_set_count(space_map::ptr sm) { - auto sm = create_sm_disk(); sm->set_count(43, 5); BOOST_CHECK_EQUAL(sm->get_count(43), 5); } -BOOST_AUTO_TEST_CASE(test_set_effects_nr_allocated) +void _test_set_affects_nr_allocated(space_map::ptr sm) { - auto sm = create_sm_disk(); for (unsigned i = 0; i < NR_BLOCKS; i++) { sm->set_count(i, 1); BOOST_CHECK_EQUAL(sm->get_nr_free(), NR_BLOCKS - i - 1); @@ -118,24 +113,143 @@ BOOST_AUTO_TEST_CASE(test_set_effects_nr_allocated) } } -BOOST_AUTO_TEST_CASE(test_reopen) +//---------------------------------------------------------------- + +#if 0 +BOOST_AUTO_TEST_CASE(reopen_an_sm) +{ + space_map::ptr sm = create_sm_disk(); +} +#endif + +BOOST_AUTO_TEST_CASE(test_disk_get_nr_blocks) +{ + space_map::ptr sm = create_sm_disk(); + _test_get_nr_blocks(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_get_nr_free) +{ + space_map::ptr sm = create_sm_disk(); + _test_get_nr_free(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_throws_no_space) +{ + space_map::ptr sm = create_sm_disk(); + _test_throws_no_space(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_inc_and_dec) +{ + space_map::ptr sm = create_sm_disk(); + _test_inc_and_dec(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_not_allocated_twice) +{ + space_map::ptr sm = create_sm_disk(); + _test_not_allocated_twice(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_set_count) +{ + space_map::ptr sm = create_sm_disk(); + _test_set_count(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_set_affects_nr_allocated) +{ + space_map::ptr sm = create_sm_disk(); + _test_set_affects_nr_allocated(sm); +} + +BOOST_AUTO_TEST_CASE(test_disk_reopen) { unsigned char buffer[128]; { - auto sm = create_sm_disk(); + persistent_space_map::ptr sm = create_sm_disk(); for (unsigned i = 0, step = 1; i < NR_BLOCKS; i += step, step++) { sm->inc(i); } + sm->commit(); BOOST_CHECK(sm->root_size() <= sizeof(buffer)); - sm->copy_root(buffer, sizeof(buffer)); } { - auto tm = create_tm(); - auto sm = persistent_data::open_disk_sm(tm, buffer); + transaction_manager::ptr tm = create_tm(); + persistent_space_map::ptr sm = persistent_data::open_disk_sm(tm, buffer); + + for (unsigned i = 0, step = 1; i < NR_BLOCKS; i += step, step++) + BOOST_CHECK_EQUAL(sm->get_count(i), 1); + } +} + +//---------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(test_metadata_get_nr_blocks) +{ + space_map::ptr sm = create_sm_metadata(); + _test_get_nr_blocks(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_get_nr_free) +{ + space_map::ptr sm = create_sm_metadata(); + _test_get_nr_free(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_throws_no_space) +{ + space_map::ptr sm = create_sm_metadata(); + _test_throws_no_space(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_inc_and_dec) +{ + space_map::ptr sm = create_sm_metadata(); + _test_inc_and_dec(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_not_allocated_twice) +{ + space_map::ptr sm = create_sm_metadata(); + _test_not_allocated_twice(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_set_count) +{ + space_map::ptr sm = create_sm_metadata(); + _test_set_count(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_set_affects_nr_allocated) +{ + space_map::ptr sm = create_sm_metadata(); + _test_set_affects_nr_allocated(sm); +} + +BOOST_AUTO_TEST_CASE(test_metadata_reopen) +{ + unsigned char buffer[128]; + + { + persistent_space_map::ptr sm = create_sm_metadata(); + for (unsigned i = 0, step = 1; i < NR_BLOCKS; i += step, step++) { + sm->inc(i); + } + sm->commit(); + + BOOST_CHECK(sm->root_size() <= sizeof(buffer)); + sm->copy_root(buffer, sizeof(buffer)); + } + + { + transaction_manager::ptr tm = create_tm(); + persistent_space_map::ptr sm = persistent_data::open_metadata_sm(tm, buffer); for (unsigned i = 0, step = 1; i < NR_BLOCKS; i += step, step++) BOOST_CHECK_EQUAL(sm->get_count(i), 1);