From 38f8195a99606deb1d3ab25ab423a28528558317 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Feb 2014 15:01:07 +0000 Subject: [PATCH 001/118] thin_show_blocks --- .gitignore | 1 + Makefile.in | 9 +- persistent-data/space-maps/disk.cc | 4 - persistent-data/space-maps/disk_structures.h | 3 + thin-provisioning/thin_show_blocks.cc | 139 +++++++++++++++++++ 5 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 thin-provisioning/thin_show_blocks.cc diff --git a/.gitignore b/.gitignore index fd2504f..c7afaf0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ thin_restore thin_repair thin_rmap thin_metadata_size +thin_show_blocks cache_check cache_dump diff --git a/Makefile.in b/Makefile.in index ca23474..70628a8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -35,7 +35,8 @@ PROGRAMS=\ thin_restore \ thin_repair \ thin_rmap \ - thin_metadata_size + thin_metadata_size \ + thin_show_blocks all: $(PROGRAMS) @@ -162,6 +163,7 @@ lib/libpdata.a: $(PDATA_OBJECTS) # Thin provisioning tools THIN_DEBUG_SOURCE=$(SOURCE) +THIN_SHOW_BLOCKS_SOURCE=$(SOURCE) THIN_DUMP_SOURCE=$(SOURCE) THIN_REPAIR_SOURCE=$(SOURCE) THIN_RESTORE_SOURCE=$(SOURCE) @@ -209,6 +211,7 @@ THIN_RMAP_SOURCE=\ thin-provisioning/superblock.cc THIN_DEBUG_OBJECTS=$(subst .cc,.o,$(THIN_DEBUG_SOURCE)) +THIN_SHOW_BLOCKS_OBJECTS=$(subst .cc,.o,$(THIN_SHOW_BLOCKS_SOURCE)) THIN_DUMP_OBJECTS=$(subst .cc,.o,$(THIN_DUMP_SOURCE)) THIN_REPAIR_OBJECTS=$(subst .cc,.o,$(THIN_REPAIR_SOURCE)) THIN_RESTORE_OBJECTS=$(subst .cc,.o,$(THIN_RESTORE_SOURCE)) @@ -219,6 +222,10 @@ thin_debug: $(THIN_DEBUG_OBJECTS) thin-provisioning/thin_debug.o @echo " [LD] $@" $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT) +thin_show_blocks: $(THIN_SHOW_BLOCKS_OBJECTS) thin-provisioning/thin_show_blocks.o + @echo " [LD] $@" + $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT) + thin_repair: $(THIN_REPAIR_OBJECTS) thin-provisioning/thin_repair.o @echo " [LD] $@" $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT) diff --git a/persistent-data/space-maps/disk.cc b/persistent-data/space-maps/disk.cc index 0c851f6..7192151 100644 --- a/persistent-data/space-maps/disk.cc +++ b/persistent-data/space-maps/disk.cc @@ -35,8 +35,6 @@ using namespace sm_disk_detail; //---------------------------------------------------------------- namespace { - uint64_t const BITMAP_CSUM_XOR = 240779; - struct bitmap_block_validator : public block_manager<>::validator { virtual void check(buffer<> const &b, block_address location) const { bitmap_header const *data = reinterpret_cast(&b); @@ -61,8 +59,6 @@ namespace { //-------------------------------- - uint64_t const INDEX_CSUM_XOR = 160478; - // FIXME: factor out the common code in these validators struct index_block_validator : public block_manager<>::validator { virtual void check(buffer<> const &b, block_address location) const { diff --git a/persistent-data/space-maps/disk_structures.h b/persistent-data/space-maps/disk_structures.h index 1429d36..e9ae8f6 100644 --- a/persistent-data/space-maps/disk_structures.h +++ b/persistent-data/space-maps/disk_structures.h @@ -110,6 +110,9 @@ namespace persistent_data { le32 not_used; le64 blocknr; } __attribute__ ((packed)); + + uint64_t const BITMAP_CSUM_XOR = 240779; + uint64_t const INDEX_CSUM_XOR = 160478; } } diff --git a/thin-provisioning/thin_show_blocks.cc b/thin-provisioning/thin_show_blocks.cc new file mode 100644 index 0000000..61debd7 --- /dev/null +++ b/thin-provisioning/thin_show_blocks.cc @@ -0,0 +1,139 @@ +#include +#include +#include +#include +#include + +#include "persistent-data/checksum.h" +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/superblock.h" +#include "version.h" + +using namespace persistent_data; +using namespace std; +using namespace sm_disk_detail; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + bool is_superblock(block_manager<>::read_ref &rr) { + using namespace superblock_detail; + + superblock_disk const *sbd = reinterpret_cast(&rr.data()); + if (to_cpu(sbd->magic_) == SUPERBLOCK_MAGIC) { + superblock sb; + superblock_traits::unpack(*sbd, sb); + cout << "metadata nr blocks: " << sb.metadata_nr_blocks_ << endl; + + return true; + } + + return false; + } + + bool is_bitmap_block(block_manager<>::read_ref &rr) { + bitmap_header const *data = reinterpret_cast(&rr.data()); + crc32c sum(BITMAP_CSUM_XOR); + sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); + return sum.get_sum() == to_cpu(data->csum); + } + + bool is_index_block(block_manager<>::read_ref &rr) { + metadata_index const *mi = reinterpret_cast(&rr.data()); + crc32c sum(INDEX_CSUM_XOR); + sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t)); + return sum.get_sum() == to_cpu(mi->csum_); + } + + bool is_btree_node(block_manager<>::read_ref &rr) { + using namespace btree_detail; + + disk_node const *data = reinterpret_cast(&rr.data()); + node_header const *n = &data->header; + crc32c sum(BTREE_CSUM_XOR); + sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + return sum.get_sum() == to_cpu(n->csum); + } + + void show_blocks(string const &dev) { + block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY); + + metadata md(bm, metadata::OPEN); + cout << "Metadata space map: nr_blocks = " << md.metadata_sm_->get_nr_blocks() + << ", nr_free_blocks = " << md.metadata_sm_->get_nr_free() + << endl; + cout << "Data space map: nr_blocks = " << md.data_sm_->get_nr_blocks() + << ", nr_free_blocks = " << md.data_sm_->get_nr_free() + << endl; + + block_address nr_blocks = bm->get_nr_blocks(); + for (block_address b = 0; b < nr_blocks; b++) { + block_manager<>::read_ref rr = bm->read_lock(b); + + if (is_superblock(rr)) + cout << b << ": superblock" << endl; + + else if (is_bitmap_block(rr)) + cout << b << ": bitmap block" << endl; + + else if (is_btree_node(rr)) + cout << b << ": btree_node" << endl; + + else + cout << b << ": unknown" << endl; + } + } + + void usage(ostream &out, string const &progname) { + out << "Usage: " << progname << " {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-V|--version}" << endl; + } +} + +//---------------------------------------------------------------- + +int main(int argc, char **argv) +{ + int c; + const char shortopts[] = "hV"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout, basename(argv[0])); + return 0; + + case 'V': + cerr << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + } + } + + if (argc == optind) { + usage(cerr, basename(argv[0])); + exit(1); + } + + try { + show_blocks(argv[optind]); + + } catch (std::exception const &e) { + cerr << e.what() << endl; + return 1; + } + + return 0; +} + +//---------------------------------------------------------------- From dbd0c650882bb80554604da3554d632b786aff13 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Aug 2015 09:41:14 +0100 Subject: [PATCH 002/118] [thin_show_duplicates] stub new command --- Makefile.in | 2 + bin/thin_show_duplicates | 1 + main.cc | 1 + thin-provisioning/commands.h | 1 + thin-provisioning/thin_show_duplicates.cc | 111 ++++++++++++++++++++++ 5 files changed, 116 insertions(+) create mode 120000 bin/thin_show_duplicates create mode 100644 thin-provisioning/thin_show_duplicates.cc diff --git a/Makefile.in b/Makefile.in index e67b300..08f895a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -88,6 +88,7 @@ SOURCE=\ thin-provisioning/thin_repair.cc \ thin-provisioning/thin_restore.cc \ thin-provisioning/thin_rmap.cc \ + thin-provisioning/thin_show_duplicates.cc \ thin-provisioning/thin_trim.cc \ thin-provisioning/xml_format.cc @@ -176,6 +177,7 @@ install: bin/pdata_tools ln -s -f pdata_tools $(BINDIR)/thin_repair ln -s -f pdata_tools $(BINDIR)/thin_restore ln -s -f pdata_tools $(BINDIR)/thin_rmap + ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates ln -s -f pdata_tools $(BINDIR)/thin_trim ln -s -f pdata_tools $(BINDIR)/thin_metadata_size ln -s -f pdata_tools $(BINDIR)/era_check diff --git a/bin/thin_show_duplicates b/bin/thin_show_duplicates new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_show_duplicates @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file diff --git a/main.cc b/main.cc index ed69ba9..ff9017b 100644 --- a/main.cc +++ b/main.cc @@ -32,6 +32,7 @@ int main(int argc, char **argv) app.add_cmd(thin_provisioning::thin_restore_cmd); app.add_cmd(thin_provisioning::thin_repair_cmd); app.add_cmd(thin_provisioning::thin_rmap_cmd); + app.add_cmd(thin_provisioning::thin_show_dups_cmd); // FIXME: convert thin_metadata_size to c++ //app.add_cmd(thin_provisioning::thin_metadata_size_cmd); diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index de63e53..65714ec 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -15,6 +15,7 @@ namespace thin_provisioning { extern base::command thin_rmap_cmd; extern base::command thin_trim_cmd; extern base::command thin_metadata_size_cmd; + extern base::command thin_show_dups_cmd; } //---------------------------------------------------------------- diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc new file mode 100644 index 0000000..06066b1 --- /dev/null +++ b/thin-provisioning/thin_show_duplicates.cc @@ -0,0 +1,111 @@ +// Copyright (C) 2015 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include + +#include "version.h" + +#include "base/application.h" +#include "base/error_state.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk.h" +#include "thin-provisioning/commands.h" +#include "thin-provisioning/device_tree.h" +#include "thin-provisioning/mapping_tree.h" +#include "thin-provisioning/superblock.h" + +using namespace base; +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + block_manager<>::ptr + open_bm(string const &path) { + block_address nr_blocks = get_nr_blocks(path); + block_manager<>::mode m = block_manager<>::READ_ONLY; + return block_manager<>::ptr(new block_manager<>(path, nr_blocks, 1, m)); + } + + transaction_manager::ptr + open_tm(block_manager<>::ptr bm) { + space_map::ptr sm(new core_map(bm->get_nr_blocks())); + sm->inc(superblock_detail::SUPERBLOCK_LOCATION); + transaction_manager::ptr tm(new transaction_manager(bm, sm)); + return tm; + } + + //-------------------------------- + + struct flags { + flags() { + } + }; + + void usage(ostream &out, string const &cmd) { + out << "Usage: " << cmd << " [options] {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-V|--version}" << endl; + } +} + +int thin_show_dups_main(int argc, char **argv) +{ + int c; + flags fs; + + char const shortopts[] = "qhV"; + option const longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout, basename(argv[0])); + return 0; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr, basename(argv[0])); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr, basename(argv[0])); + exit(1); + } + + return 0; +} + +base::command thin_provisioning::thin_show_dups_cmd("thin_show_duplicates", thin_show_dups_main); + +//---------------------------------------------------------------- From 59a622670c775bba1348936ee3eb5bef1814575e Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Aug 2015 12:32:57 +0100 Subject: [PATCH 003/118] [thin_show_duplicates] wip --- persistent-data/block.tcc | 1 + persistent-data/file_utils.cc | 4 +- persistent-data/file_utils.h | 2 +- thin-provisioning/thin_show_duplicates.cc | 125 +++++++++++++++++++++- 4 files changed, 124 insertions(+), 8 deletions(-) diff --git a/persistent-data/block.tcc b/persistent-data/block.tcc index 6e47a91..0824bf3 100644 --- a/persistent-data/block.tcc +++ b/persistent-data/block.tcc @@ -223,6 +223,7 @@ namespace persistent_data { unsigned max_concurrent_blocks, mode m, bool excl) + // FIXME: * BlockSize ? : fd_(open_or_create_block_file(path, nr_blocks * BlockSize, m, excl)), bc_(fd_, BlockSize >> SECTOR_SHIFT, nr_blocks, 1024u * 1024u * 16), superblock_ref_count_(0) diff --git a/persistent-data/file_utils.cc b/persistent-data/file_utils.cc index 2467079..b00acf0 100644 --- a/persistent-data/file_utils.cc +++ b/persistent-data/file_utils.cc @@ -12,7 +12,7 @@ using namespace base; //---------------------------------------------------------------- persistent_data::block_address -persistent_data::get_nr_blocks(string const &path) +persistent_data::get_nr_blocks(string const &path, sector_t block_size) { using namespace persistent_data; @@ -39,7 +39,7 @@ persistent_data::get_nr_blocks(string const &path) throw runtime_error("ioctl BLKGETSIZE64 failed"); } ::close(fd); - nr_blocks = div_down(nr_blocks, MD_BLOCK_SIZE); + nr_blocks = div_down(nr_blocks, block_size); } else // FIXME: needs a better message throw runtime_error("bad path"); diff --git a/persistent-data/file_utils.h b/persistent-data/file_utils.h index fcf203d..e641b7a 100644 --- a/persistent-data/file_utils.h +++ b/persistent-data/file_utils.h @@ -9,7 +9,7 @@ // FIXME: move to a different unit namespace persistent_data { - persistent_data::block_address get_nr_blocks(string const &path); + persistent_data::block_address get_nr_blocks(string const &path, sector_t block_size = MD_BLOCK_SIZE); block_manager<>::ptr open_bm(std::string const &dev_path, block_manager<>::mode m, bool excl = true); diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 06066b1..7666c0e 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -32,6 +32,9 @@ #include "thin-provisioning/mapping_tree.h" #include "thin-provisioning/superblock.h" +#include +#include + using namespace base; using namespace std; using namespace thin_provisioning; @@ -57,14 +60,126 @@ namespace { //-------------------------------- struct flags { - flags() { + flags() + : block_size(512 * 1024), + cache_mem(64 * 1024 * 1024) { } + + unsigned block_size; + unsigned cache_mem; }; + int open_file(string const &path) { + int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT | O_EXCL, 0666); + if (fd < 0) + syscall_failed("open", + "Note: you cannot run this tool with these options on live metadata."); + + return fd; + } + + class duplicate_counter { + public: + duplicate_counter(block_address nr_blocks) + : counts_(nr_blocks), + total_dups_(0) { + } + + void add_duplicate(block_address b1, block_address b2) { + // cout << "block " << b2 << " is a duplicate of " << b1 << "\n"; + total_dups_++; + counts_[b1]++; + } + + block_address get_total() const { + return total_dups_; + } + + private: + vector counts_; + block_address total_dups_; + }; + + class duplicate_detector { + public: + duplicate_detector(unsigned block_size, block_address nr_blocks) + : block_size_(block_size), + results_(nr_blocks) { + } + + void examine(block_cache::block const &b) { + digestor_.reset(); + digestor_.process_bytes(b.get_data(), block_size_); + unsigned int digest[5]; + digestor_.get_digest(digest); + + // hack + vector v(5); + for (unsigned i = 0; i < 5; i++) + v[i] = digest[i]; + + fingerprint_map::const_iterator it = fm_.find(v); + if (it != fm_.end()) { + results_.add_duplicate(it->second, b.get_index()); + } else + fm_.insert(make_pair(v, b.get_index())); + } + + block_address get_total_duplicates() const { + return results_.get_total(); + } + + private: + typedef map, block_address> fingerprint_map; + + unsigned block_size_; + boost::uuids::detail::sha1 digestor_; + fingerprint_map fm_; + duplicate_counter results_; + }; + + int show_dups(string const &path, flags const &fs) { + cerr << "path = " << path << "\n"; + block_address nr_blocks = get_nr_blocks(path, fs.block_size); + cerr << "nr_blocks = " << nr_blocks << "\n"; + + // The cache uses a LRU eviction policy, which plays badly + // with a sequential read. So we can't prefetch all the + // blocks. + + // FIXME: add MRU policy to cache + unsigned cache_blocks = (fs.cache_mem / fs.block_size) / 2; + int fd = open_file(path); + sector_t block_sectors = fs.block_size / 512; + block_cache cache(fd, block_sectors, nr_blocks, fs.cache_mem); + validator::ptr v(new bcache::noop_validator()); + + duplicate_detector detector(fs.block_size, nr_blocks); + + // warm up the cache + for (block_address i = 0; i < cache_blocks; i++) + cache.prefetch(i); + + for (block_address i = 0; i < nr_blocks; i++) { + block_cache::block &b = cache.get(i, 0, v); + block_address prefetch = i + cache_blocks; + if (prefetch < nr_blocks) + cache.prefetch(prefetch); + + detector.examine(b); + b.put(); + } + + cout << "total dups: " << detector.get_total_duplicates() << endl; + + return 0; + } + void usage(ostream &out, string const &cmd) { - out << "Usage: " << cmd << " [options] {device|file}" << endl - << "Options:" << endl - << " {-h|--help}" << endl + out << "Usage: " << cmd << " [options] {device|file}\n" + << "Options:\n" + << " {--block-sectors} \n" + << " {-h|--help}\n" << " {-V|--version}" << endl; } } @@ -103,7 +218,7 @@ int thin_show_dups_main(int argc, char **argv) exit(1); } - return 0; + return show_dups(argv[optind], fs); } base::command thin_provisioning::thin_show_dups_cmd("thin_show_duplicates", thin_show_dups_main); From 25f4f23e4202dd31866d4832a1e5b3424daccfb2 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Aug 2015 12:44:07 +0100 Subject: [PATCH 004/118] [file_utils] fix bug in get_nr_blocks. Introduced in previous patch --- persistent-data/file_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistent-data/file_utils.cc b/persistent-data/file_utils.cc index b00acf0..88ee945 100644 --- a/persistent-data/file_utils.cc +++ b/persistent-data/file_utils.cc @@ -24,7 +24,7 @@ persistent_data::get_nr_blocks(string const &path, sector_t block_size) throw runtime_error("Couldn't stat dev path"); if (S_ISREG(info.st_mode) && info.st_size) - nr_blocks = div_up(info.st_size, MD_BLOCK_SIZE); + nr_blocks = div_up(info.st_size, block_size); else if (S_ISBLK(info.st_mode)) { // To get the size of a block device we need to From 519cbfd855178fa8fcb52894e53a129ad63357ca Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Aug 2015 12:53:11 +0100 Subject: [PATCH 005/118] [thin_show_duplicates] add a progress bar --- thin-provisioning/thin_show_duplicates.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 7666c0e..f9aef87 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -24,6 +24,7 @@ #include "base/application.h" #include "base/error_state.h" +#include "base/progress_monitor.h" #include "persistent-data/file_utils.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/space-maps/disk.h" @@ -140,6 +141,7 @@ namespace { int show_dups(string const &path, flags const &fs) { cerr << "path = " << path << "\n"; + cerr << "block size = " << fs.block_size << "\n"; block_address nr_blocks = get_nr_blocks(path, fs.block_size); cerr << "nr_blocks = " << nr_blocks << "\n"; @@ -160,6 +162,8 @@ namespace { for (block_address i = 0; i < cache_blocks; i++) cache.prefetch(i); + auto_ptr pbar = create_progress_bar("Examining data"); + for (block_address i = 0; i < nr_blocks; i++) { block_cache::block &b = cache.get(i, 0, v); block_address prefetch = i + cache_blocks; @@ -168,9 +172,11 @@ namespace { detector.examine(b); b.put(); + + pbar->update_percent(i * 100 / nr_blocks); } - cout << "total dups: " << detector.get_total_duplicates() << endl; + cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; return 0; } From 929a824184134fb63d8c7eefa05be5bbeb336f7c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Aug 2015 13:07:56 +0100 Subject: [PATCH 006/118] [thin_show_duplicates] add --block-sectors switch --- thin-provisioning/thin_show_duplicates.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index f9aef87..04f8150 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -34,6 +34,7 @@ #include "thin-provisioning/superblock.h" #include +#include #include using namespace base; @@ -58,6 +59,19 @@ namespace { return tm; } + uint64_t parse_int(string const &str, string const &desc) { + try { + return boost::lexical_cast(str); + + } catch (...) { + ostringstream out; + out << "Couldn't parse " << desc << ": '" << str << "'"; + exit(1); + } + + return 0; // never get here + } + //-------------------------------- struct flags { @@ -197,6 +211,7 @@ int thin_show_dups_main(int argc, char **argv) char const shortopts[] = "qhV"; option const longopts[] = { + { "block-sectors", required_argument, NULL, 1}, { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } @@ -212,6 +227,10 @@ int thin_show_dups_main(int argc, char **argv) cout << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; + case 1: + fs.block_size = 512 * parse_int(optarg, "block sectors"); + break; + default: usage(cerr, basename(argv[0])); return 1; From 94636b63d7c2d2d7ae708b64bab84aa7a025432f Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 19 Aug 2015 13:46:02 +0100 Subject: [PATCH 007/118] [thin_show_duplicates] print out the percentage of duplicates --- thin-provisioning/thin_show_duplicates.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 04f8150..12d3490 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -191,6 +191,7 @@ namespace { } cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; + cout << (detector.get_total_duplicates() * 100) / nr_blocks << "% duplicates\n"; return 0; } From 5f11f5af9993cfa9f8e1bb80dd1f6b4cdef5801d Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 20 Aug 2015 11:12:53 +0100 Subject: [PATCH 008/118] [progress_bar] Tidy up the appearance when at 100% --- base/progress_monitor.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/base/progress_monitor.cc b/base/progress_monitor.cc index 1d88302..a33e08c 100644 --- a/base/progress_monitor.cc +++ b/base/progress_monitor.cc @@ -31,17 +31,22 @@ namespace { if (nr_equals < progress_width_) cout << '>'; + else + cout << "="; for (unsigned i = 0; i < nr_spaces; i++) cout << ' '; - cout << "] " << spinner_char() << " " << p << "%\r" << flush; + cout << "] " << spinner_char(p) << " " << p << "%\r" << flush; spinner_++; } private: - char spinner_char() const { + char spinner_char(unsigned p) const { + if (p == 100) + return ' '; + char cs[] = {'|', '/', '-', '\\'}; unsigned index = spinner_ % sizeof(cs); From d954f230fa1fb77c5934734f83eaa44140f42e5e Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 21 Aug 2015 13:10:49 +0100 Subject: [PATCH 009/118] [thin_show_duplicates] wip --- thin-provisioning/thin_show_duplicates.cc | 181 ++++++++++++++++++++-- 1 file changed, 168 insertions(+), 13 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 12d3490..d743685 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -25,6 +25,7 @@ #include "base/application.h" #include "base/error_state.h" #include "base/progress_monitor.h" +#include "persistent-data/data-structures/btree_damage_visitor.h" #include "persistent-data/file_utils.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/space-maps/disk.h" @@ -32,18 +33,27 @@ #include "thin-provisioning/device_tree.h" #include "thin-provisioning/mapping_tree.h" #include "thin-provisioning/superblock.h" +#include "thin-provisioning/rmap_visitor.h" #include #include +#include #include using namespace base; +using namespace boost; +using namespace persistent_data; using namespace std; using namespace thin_provisioning; //---------------------------------------------------------------- namespace { + bool factor_of(block_address f, block_address n) { + cerr << n << " % " << f << "\n"; + return (n % f) == 0; + } + block_manager<>::ptr open_bm(string const &path) { block_address nr_blocks = get_nr_blocks(path); @@ -74,13 +84,21 @@ namespace { //-------------------------------- + struct data_block { + block_address begin, end; + void *data; + }; + + //-------------------------------- + struct flags { flags() - : block_size(512 * 1024), - cache_mem(64 * 1024 * 1024) { + : cache_mem(64 * 1024 * 1024) { } - unsigned block_size; + string data_dev; + optional metadata_dev; + optional block_size; unsigned cache_mem; }; @@ -93,6 +111,38 @@ namespace { return fd; } + + // FIXME: introduce abstraction for a stream of segments + + using namespace mapping_tree_detail; + + typedef rmap_visitor::region region; + typedef rmap_visitor::rmap_region rmap_region; + + class damage_visitor { + public: + virtual void visit(btree_path const &path, btree_detail::damage const &d) { + throw std::runtime_error("damage in mapping tree, please run thin_check"); + } + }; + + // FIXME: too big to return by value + vector read_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks) { + damage_visitor dv; + rmap_visitor rv; + + mapping_tree mtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::block_traits::ref_counter(tm->get_sm())); + + rv.add_data_region(rmap_visitor::region(0, nr_blocks)); + + btree_visit_values(mtree, rv, dv); + rv.complete(); + cerr << "rmap size: " << rv.get_rmap().size() << "\n"; + return rv.get_rmap(); + } + class duplicate_counter { public: duplicate_counter(block_address nr_blocks) @@ -101,7 +151,6 @@ namespace { } void add_duplicate(block_address b1, block_address b2) { - // cout << "block " << b2 << " is a duplicate of " << b1 << "\n"; total_dups_++; counts_[b1]++; } @@ -153,10 +202,98 @@ namespace { duplicate_counter results_; }; - int show_dups(string const &path, flags const &fs) { - cerr << "path = " << path << "\n"; + int show_dups_pool(flags const &fs) { + block_manager<>::ptr bm = open_bm(*fs.metadata_dev); + transaction_manager::ptr tm = open_tm(bm); + superblock_detail::superblock sb = read_superblock(bm); + + block_address block_size = sb.data_block_size_ * 512; +#if 0 + if (fs.block_size) { + if (!factor_of(*fs.block_size, sb.data_block_size_ * 512)) + throw runtime_error("specified block size must be a factor of the pool block size."); + + block_size = *fs.block_size; + } +#endif + + cerr << "path = " << fs.data_dev << "\n"; + cerr << "block size = " << block_size << "\n"; + block_address nr_blocks = get_nr_blocks(fs.data_dev, block_size); + cerr << "nr_blocks = " << nr_blocks << "\n"; + + cerr << "reading rmap..."; + vector rmap = read_rmap(tm, sb, nr_blocks); + cerr << "done\n"; + + uint32_t const UNMAPPED = -1; + vector block_to_thin(nr_blocks, UNMAPPED); + vector::const_iterator it; + set thins; + block_address nr_mapped = 0; + for (it = rmap.begin(); it != rmap.end(); ++it) { + rmap_region const &r = *it; + for (block_address b = r.data_begin; b != r.data_end; b++) + if (block_to_thin[b] == UNMAPPED) { + nr_mapped++; + block_to_thin[b] = r.thin_dev; + } + thins.insert(r.thin_dev); + } + cerr << nr_mapped << " mapped blocks\n"; + + cerr << "there are " << thins.size() << " thin devices\n"; + + // The cache uses a LRU eviction policy, which plays badly + // with a sequential read. So we can't prefetch all the + // blocks. + + // FIXME: add MRU policy to cache + unsigned cache_blocks = (fs.cache_mem / block_size) / 2; + int fd = open_file(fs.data_dev); + sector_t block_sectors = block_size / 512; + block_cache cache(fd, block_sectors, nr_blocks, fs.cache_mem); + validator::ptr v(new bcache::noop_validator()); + + duplicate_detector detector(block_size, nr_blocks); + + // warm up the cache + for (block_address i = 0; i < cache_blocks; i++) + cache.prefetch(i); + + auto_ptr pbar = create_progress_bar("Examining data"); + + for (block_address i = 0; i < nr_blocks; i++) { + if (block_to_thin[i] == UNMAPPED) + continue; + + block_cache::block &b = cache.get(i, 0, v); + block_address prefetch = i + cache_blocks; + if (prefetch < nr_blocks) + cache.prefetch(prefetch); + + detector.examine(b); + b.put(); + + if (!(i & 127)) + pbar->update_percent(i * 100 / nr_blocks); + } + pbar->update_percent(100); + + cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; + cout << (detector.get_total_duplicates() * 100) / nr_mapped << "% duplicates\n"; + + return 0; + } + + int show_dups_linear(flags const &fs) { + if (!fs.block_size) + // FIXME: this check should be moved to the switch parsing + throw runtime_error("--block-sectors or --metadata-dev must be supplied"); + + cerr << "path = " << fs.data_dev << "\n"; cerr << "block size = " << fs.block_size << "\n"; - block_address nr_blocks = get_nr_blocks(path, fs.block_size); + block_address nr_blocks = get_nr_blocks(fs.data_dev, *fs.block_size); cerr << "nr_blocks = " << nr_blocks << "\n"; // The cache uses a LRU eviction policy, which plays badly @@ -164,13 +301,13 @@ namespace { // blocks. // FIXME: add MRU policy to cache - unsigned cache_blocks = (fs.cache_mem / fs.block_size) / 2; - int fd = open_file(path); - sector_t block_sectors = fs.block_size / 512; + unsigned cache_blocks = (fs.cache_mem / *fs.block_size) / 2; + int fd = open_file(fs.data_dev); + sector_t block_sectors = *fs.block_size / 512; block_cache cache(fd, block_sectors, nr_blocks, fs.cache_mem); validator::ptr v(new bcache::noop_validator()); - duplicate_detector detector(fs.block_size, nr_blocks); + duplicate_detector detector(*fs.block_size, nr_blocks); // warm up the cache for (block_address i = 0; i < cache_blocks; i++) @@ -189,6 +326,7 @@ namespace { pbar->update_percent(i * 100 / nr_blocks); } + pbar->update_percent(100); cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; cout << (detector.get_total_duplicates() * 100) / nr_blocks << "% duplicates\n"; @@ -196,10 +334,20 @@ namespace { return 0; } + int show_dups(flags const &fs) { + if (fs.metadata_dev) + return show_dups_pool(fs); + else { + cerr << "No metadata device provided, so treating data device as a linear device\n"; + return show_dups_linear(fs); + } + } + void usage(ostream &out, string const &cmd) { out << "Usage: " << cmd << " [options] {device|file}\n" << "Options:\n" << " {--block-sectors} \n" + << " {--metadata-dev} \n" << " {-h|--help}\n" << " {-V|--version}" << endl; } @@ -213,6 +361,7 @@ int thin_show_dups_main(int argc, char **argv) char const shortopts[] = "qhV"; option const longopts[] = { { "block-sectors", required_argument, NULL, 1}, + { "metadata-dev", required_argument, NULL, 2}, { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } @@ -232,6 +381,10 @@ int thin_show_dups_main(int argc, char **argv) fs.block_size = 512 * parse_int(optarg, "block sectors"); break; + case 2: + fs.metadata_dev = optarg; + break; + default: usage(cerr, basename(argv[0])); return 1; @@ -239,12 +392,14 @@ int thin_show_dups_main(int argc, char **argv) } if (argc == optind) { - cerr << "No input file provided." << endl; + cerr << "No data device/file provided." << endl; usage(cerr, basename(argv[0])); exit(1); } - return show_dups(argv[optind], fs); + fs.data_dev = argv[optind]; + + return show_dups(fs); } base::command thin_provisioning::thin_show_dups_cmd("thin_show_duplicates", thin_show_dups_main); From c8d3ce6af5f7ab60b322800f3effe590e15ee467 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 24 Aug 2015 11:18:31 +0100 Subject: [PATCH 010/118] [thin_show_duplicates] start factoring out a chunk_stream abstraction --- block-cache/block_cache.h | 39 +++++++ thin-provisioning/thin_show_duplicates.cc | 118 +++++++++++++++++++--- 2 files changed, 144 insertions(+), 13 deletions(-) diff --git a/block-cache/block_cache.h b/block-cache/block_cache.h index 4bc6667..008af15 100644 --- a/block-cache/block_cache.h +++ b/block-cache/block_cache.h @@ -12,6 +12,7 @@ #include #include #include +#include //---------------------------------------------------------------- @@ -112,6 +113,44 @@ namespace bcache { validator::ptr v_; }; + class auto_block { + public: + auto_block() + : b_(0) { + } + + auto_block(block &b) + : b_(&b) { + } + + ~auto_block() { + put(); + } + + auto_block &operator =(block &b) { + put(); + b_ = &b; + return *this; + } + + void *get_data() const { + if (b_) + return b_->get_data(); + + throw std::runtime_error("auto_block not set"); + } + + private: + void put() { + if (b_) { + b_->put(); + b_ = 0; + } + } + + block *b_; + }; + //-------------------------------- block_cache(int fd, sector_t block_size, diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index d743685..9be5c07 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include using namespace base; @@ -54,6 +55,15 @@ namespace { return (n % f) == 0; } + int open_file(string const &path) { + int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT | O_EXCL, 0666); + if (fd < 0) + syscall_failed("open", + "Note: you cannot run this tool with these options on live metadata."); + + return fd; + } + block_manager<>::ptr open_bm(string const &path) { block_address nr_blocks = get_nr_blocks(path); @@ -84,9 +94,97 @@ namespace { //-------------------------------- - struct data_block { - block_address begin, end; - void *data; + // Once we start using variable sized blocks we will find we want + // to examine data that crosses cache block boundaries. So a block + // to be examined can be composed of multiple chunks of memory. + + struct mem { + mem(void *b, void *e) + : begin(b), + end(e) { + } + + void *begin, *end; + }; + + struct chunk { + sector_t offset_sectors_; + deque mem_; + }; + + class chunk_stream { + public: + virtual ~chunk_stream() {} + + virtual void rewind() = 0; + virtual bool advance() = 0; + virtual chunk const &get() const = 0; + }; + + class cache_stream : public chunk_stream { + public: + cache_stream(string const &path, + block_address block_size, + size_t cache_mem) + : block_size_(block_size), + nr_blocks_(get_nr_blocks(path, block_size)), + + // hack because cache uses LRU rather than MRU + cache_blocks_((cache_mem / block_size) / 2u), + fd_(open_file(path)), + v_(new bcache::noop_validator()), + cache_(new block_cache(fd_, block_size / 512, nr_blocks_, cache_mem)), + current_index_(0) { + load(0); + } + + virtual void rewind() { + load(0); + } + + virtual bool advance() { + if (current_index_ >= nr_blocks_) + return false; + + current_index_++; + + load(current_index_); + return true; + } + + virtual chunk const &get() const { + return current_chunk_; + } + + private: + void load(block_address b) { + current_index_ = b; + current_block_ = cache_->get(current_index_, 0, v_); + + current_chunk_.offset_sectors_ = (b * block_size_) / 512; + current_chunk_.mem_.clear(); + current_chunk_.mem_.push_back(mem(current_block_.get_data(), + current_block_.get_data() + block_size_)); + } + + block_address block_size_; + block_address nr_blocks_; + block_address cache_blocks_; + int fd_; + validator::ptr v_; + auto_ptr cache_; + + block_address current_index_; + block_cache::auto_block current_block_; + chunk current_chunk_; + }; + + class fixed_block_stream : public chunk_stream { + public: + }; + + class variable_size_stream : public chunk_stream { + }; //-------------------------------- @@ -102,16 +200,6 @@ namespace { unsigned cache_mem; }; - int open_file(string const &path) { - int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT | O_EXCL, 0666); - if (fd < 0) - syscall_failed("open", - "Note: you cannot run this tool with these options on live metadata."); - - return fd; - } - - // FIXME: introduce abstraction for a stream of segments using namespace mapping_tree_detail; @@ -217,6 +305,10 @@ namespace { } #endif + { + cache_stream(fs.data_dev, block_size, fs.cache_mem); + } + cerr << "path = " << fs.data_dev << "\n"; cerr << "block size = " << block_size << "\n"; block_address nr_blocks = get_nr_blocks(fs.data_dev, block_size); From ac4104d063e518d183ec032cd1b9ddac4549f28f Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 24 Aug 2015 11:24:55 +0100 Subject: [PATCH 011/118] add prefetching --- thin-provisioning/thin_show_duplicates.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 9be5c07..3142e62 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -136,6 +136,8 @@ namespace { cache_(new block_cache(fd_, block_size / 512, nr_blocks_, cache_mem)), current_index_(0) { load(0); + for (block_address i = 1; i < min(cache_blocks_, nr_blocks_); i++) + cache_->prefetch(i); } virtual void rewind() { @@ -165,6 +167,9 @@ namespace { current_chunk_.mem_.clear(); current_chunk_.mem_.push_back(mem(current_block_.get_data(), current_block_.get_data() + block_size_)); + + if (current_index_ + cache_blocks_ < nr_blocks_) + cache_->prefetch(current_index_ + cache_blocks_); } block_address block_size_; From 46fe4525bb900e8b6cbbf10fa9f62f16bb4f1402 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 24 Aug 2015 14:29:06 +0100 Subject: [PATCH 012/118] [thin_show_dups] factor out a pool stream --- thin-provisioning/thin_show_duplicates.cc | 259 +++++++++++++--------- 1 file changed, 155 insertions(+), 104 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 3142e62..96d1a23 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -99,15 +99,17 @@ namespace { // to be examined can be composed of multiple chunks of memory. struct mem { - mem(void *b, void *e) + mem(uint8_t *b, uint8_t *e) : begin(b), end(e) { } - void *begin, *end; + uint8_t *begin, *end; }; struct chunk { + // FIXME: switch to bytes rather than sectors + // FIXME: add length too sector_t offset_sectors_; deque mem_; }; @@ -116,8 +118,10 @@ namespace { public: virtual ~chunk_stream() {} + virtual block_address nr_chunks() const = 0; virtual void rewind() = 0; - virtual bool advance() = 0; + virtual bool advance(block_address count = 1ull) = 0; + virtual block_address index() const = 0; virtual chunk const &get() const = 0; }; @@ -140,20 +144,28 @@ namespace { cache_->prefetch(i); } + virtual block_address nr_chunks() const { + return nr_blocks_; + } + virtual void rewind() { load(0); } - virtual bool advance() { - if (current_index_ >= nr_blocks_) + virtual bool advance(block_address count = 1ull) { + if (current_index_ + count >= nr_blocks_) return false; - current_index_++; + current_index_ += count; load(current_index_); return true; } + virtual block_address index() const { + return current_index_; + } + virtual chunk const &get() const { return current_chunk_; } @@ -165,8 +177,8 @@ namespace { current_chunk_.offset_sectors_ = (b * block_size_) / 512; current_chunk_.mem_.clear(); - current_chunk_.mem_.push_back(mem(current_block_.get_data(), - current_block_.get_data() + block_size_)); + current_chunk_.mem_.push_back(mem(static_cast(current_block_.get_data()), + static_cast(current_block_.get_data()) + block_size_)); if (current_index_ + cache_blocks_ < nr_blocks_) cache_->prefetch(current_index_ + cache_blocks_); @@ -184,12 +196,112 @@ namespace { chunk current_chunk_; }; - class fixed_block_stream : public chunk_stream { + //-------------------------------- + + typedef rmap_visitor::region region; + typedef rmap_visitor::rmap_region rmap_region; + + uint32_t const UNMAPPED = -1; + + class pool_stream : public chunk_stream { public: - }; - class variable_size_stream : public chunk_stream { + pool_stream(cache_stream &stream, + transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks) + : stream_(stream), + block_to_thin_(stream.nr_chunks(), UNMAPPED), + nr_mapped_(0) { + init_rmap(tm, sb, nr_blocks); + } + block_address nr_chunks() const { + return stream_.nr_chunks(); + } + + void rewind() { + stream_.rewind(); + } + + bool advance(block_address count = 1ull) { + while (count--) + if (!advance_one()) + return false; + + return true; + } + + block_address index() const { + return stream_.index(); + } + + chunk const &get() const { + return stream_.get(); + } + + private: + class damage_visitor { + public: + virtual void visit(btree_path const &path, btree_detail::damage const &d) { + throw std::runtime_error("damage in mapping tree, please run thin_check"); + } + }; + + // FIXME: too big to return by value + vector read_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks) { + damage_visitor dv; + rmap_visitor rv; + + mapping_tree mtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::block_traits::ref_counter(tm->get_sm())); + + rv.add_data_region(rmap_visitor::region(0, nr_blocks)); + + btree_visit_values(mtree, rv, dv); + rv.complete(); + cerr << "rmap size: " << rv.get_rmap().size() << "\n"; + return rv.get_rmap(); + } + + void init_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks) { + cerr << "reading rmap..."; + vector rmap = read_rmap(tm, sb, nr_blocks); + cerr << "done\n"; + + vector::const_iterator it; + set thins; + for (it = rmap.begin(); it != rmap.end(); ++it) { + rmap_region const &r = *it; + for (block_address b = r.data_begin; b != r.data_end; b++) + if (block_to_thin_[b] == UNMAPPED) { + nr_mapped_++; + block_to_thin_[b] = r.thin_dev; + } + thins.insert(r.thin_dev); + } + + cerr << nr_mapped_ << " mapped blocks\n"; + cerr << "there are " << thins.size() << " thin devices\n"; + } + + bool advance_one() { + block_address new_index = index() + 1; + + while (block_to_thin_[new_index] == UNMAPPED && + new_index < nr_chunks()) + new_index++; + + if (new_index >= nr_chunks()) + return false; + + return stream_.advance(new_index - index()); + } + + cache_stream &stream_; + vector block_to_thin_; + block_address nr_mapped_; }; //-------------------------------- @@ -209,33 +321,6 @@ namespace { using namespace mapping_tree_detail; - typedef rmap_visitor::region region; - typedef rmap_visitor::rmap_region rmap_region; - - class damage_visitor { - public: - virtual void visit(btree_path const &path, btree_detail::damage const &d) { - throw std::runtime_error("damage in mapping tree, please run thin_check"); - } - }; - - // FIXME: too big to return by value - vector read_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, - block_address nr_blocks) { - damage_visitor dv; - rmap_visitor rv; - - mapping_tree mtree(*tm, sb.data_mapping_root_, - mapping_tree_detail::block_traits::ref_counter(tm->get_sm())); - - rv.add_data_region(rmap_visitor::region(0, nr_blocks)); - - btree_visit_values(mtree, rv, dv); - rv.complete(); - cerr << "rmap size: " << rv.get_rmap().size() << "\n"; - return rv.get_rmap(); - } - class duplicate_counter { public: duplicate_counter(block_address nr_blocks) @@ -264,6 +349,7 @@ namespace { results_(nr_blocks) { } + // FIXME: remove void examine(block_cache::block const &b) { digestor_.reset(); digestor_.process_bytes(b.get_data(), block_size_); @@ -282,6 +368,28 @@ namespace { fm_.insert(make_pair(v, b.get_index())); } + void examine(chunk const &c) { + digestor_.reset(); + + for (deque::const_iterator it = c.mem_.begin(); it != c.mem_.end(); it++) + digestor_.process_bytes(it->begin, it->end - it->begin); + + unsigned int digest[5]; + digestor_.get_digest(digest); + + // hack + vector v(5); + for (unsigned i = 0; i < 5; i++) + v[i] = digest[i]; + + fingerprint_map::const_iterator it = fm_.find(v); + block_address index = (c.offset_sectors_ * 512) / block_size_; + if (it != fm_.end()) { + results_.add_duplicate(it->second, index); + } else + fm_.insert(make_pair(v, index)); + } + block_address get_total_duplicates() const { return results_.get_total(); } @@ -299,86 +407,29 @@ namespace { block_manager<>::ptr bm = open_bm(*fs.metadata_dev); transaction_manager::ptr tm = open_tm(bm); superblock_detail::superblock sb = read_superblock(bm); - block_address block_size = sb.data_block_size_ * 512; -#if 0 - if (fs.block_size) { - if (!factor_of(*fs.block_size, sb.data_block_size_ * 512)) - throw runtime_error("specified block size must be a factor of the pool block size."); - - block_size = *fs.block_size; - } -#endif - - { - cache_stream(fs.data_dev, block_size, fs.cache_mem); - } + block_address nr_blocks = get_nr_blocks(fs.data_dev, block_size); cerr << "path = " << fs.data_dev << "\n"; cerr << "block size = " << block_size << "\n"; - block_address nr_blocks = get_nr_blocks(fs.data_dev, block_size); cerr << "nr_blocks = " << nr_blocks << "\n"; - cerr << "reading rmap..."; - vector rmap = read_rmap(tm, sb, nr_blocks); - cerr << "done\n"; - - uint32_t const UNMAPPED = -1; - vector block_to_thin(nr_blocks, UNMAPPED); - vector::const_iterator it; - set thins; - block_address nr_mapped = 0; - for (it = rmap.begin(); it != rmap.end(); ++it) { - rmap_region const &r = *it; - for (block_address b = r.data_begin; b != r.data_end; b++) - if (block_to_thin[b] == UNMAPPED) { - nr_mapped++; - block_to_thin[b] = r.thin_dev; - } - thins.insert(r.thin_dev); - } - cerr << nr_mapped << " mapped blocks\n"; - - cerr << "there are " << thins.size() << " thin devices\n"; - - // The cache uses a LRU eviction policy, which plays badly - // with a sequential read. So we can't prefetch all the - // blocks. - - // FIXME: add MRU policy to cache - unsigned cache_blocks = (fs.cache_mem / block_size) / 2; - int fd = open_file(fs.data_dev); - sector_t block_sectors = block_size / 512; - block_cache cache(fd, block_sectors, nr_blocks, fs.cache_mem); - validator::ptr v(new bcache::noop_validator()); + cache_stream stream(fs.data_dev, block_size, fs.cache_mem); + pool_stream pstream(stream, tm, sb, nr_blocks); duplicate_detector detector(block_size, nr_blocks); - - // warm up the cache - for (block_address i = 0; i < cache_blocks; i++) - cache.prefetch(i); - auto_ptr pbar = create_progress_bar("Examining data"); - for (block_address i = 0; i < nr_blocks; i++) { - if (block_to_thin[i] == UNMAPPED) - continue; + do { + chunk const &c = pstream.get(); + detector.examine(c); + pbar->update_percent((pstream.index() * 100) / pstream.nr_chunks()); - block_cache::block &b = cache.get(i, 0, v); - block_address prefetch = i + cache_blocks; - if (prefetch < nr_blocks) - cache.prefetch(prefetch); - - detector.examine(b); - b.put(); - - if (!(i & 127)) - pbar->update_percent(i * 100 / nr_blocks); - } + } while (pstream.advance()); pbar->update_percent(100); cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; - cout << (detector.get_total_duplicates() * 100) / nr_mapped << "% duplicates\n"; +// cout << (detector.get_total_duplicates() * 100) / nr_mapped_ << "% duplicates\n"; return 0; } From 3470ede50b256a3911fafc2b8df24aa9852c14e9 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 24 Aug 2015 16:55:53 +0100 Subject: [PATCH 013/118] [thin_show_dups] pull the various streams out to their own files --- Makefile.in | 2 + thin-provisioning/cache_stream.cc | 91 +++++++++ thin-provisioning/cache_stream.h | 41 ++++ thin-provisioning/chunk_stream.h | 60 ++++++ thin-provisioning/pool_stream.cc | 146 ++++++++++++++ thin-provisioning/pool_stream.h | 60 ++++++ thin-provisioning/thin_show_duplicates.cc | 229 +--------------------- 7 files changed, 406 insertions(+), 223 deletions(-) create mode 100644 thin-provisioning/cache_stream.cc create mode 100644 thin-provisioning/cache_stream.h create mode 100644 thin-provisioning/chunk_stream.h create mode 100644 thin-provisioning/pool_stream.cc create mode 100644 thin-provisioning/pool_stream.h diff --git a/Makefile.in b/Makefile.in index 08f895a..0dd6c99 100644 --- a/Makefile.in +++ b/Makefile.in @@ -71,12 +71,14 @@ SOURCE=\ persistent-data/space_map.cc \ persistent-data/transaction_manager.cc \ persistent-data/validators.cc \ + thin-provisioning/cache_stream.cc \ thin-provisioning/device_tree.cc \ thin-provisioning/human_readable_format.cc \ thin-provisioning/mapping_tree.cc \ thin-provisioning/metadata.cc \ thin-provisioning/metadata_checker.cc \ thin-provisioning/metadata_dumper.cc \ + thin-provisioning/pool_stream.cc \ thin-provisioning/restore_emitter.cc \ thin-provisioning/rmap_visitor.cc \ thin-provisioning/superblock.cc \ diff --git a/thin-provisioning/cache_stream.cc b/thin-provisioning/cache_stream.cc new file mode 100644 index 0000000..956fc4e --- /dev/null +++ b/thin-provisioning/cache_stream.cc @@ -0,0 +1,91 @@ +#include "thin-provisioning/cache_stream.h" +#include "persistent-data/file_utils.h" + +using namespace thin_provisioning; +using namespace std; +using namespace persistent_data; + +//---------------------------------------------------------------- + +namespace { + int open_file(string const &path) { + int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT | O_EXCL, 0666); + if (fd < 0) + syscall_failed("open", + "Note: you cannot run this tool with these options on live metadata."); + + return fd; + } +} + +//---------------------------------------------------------------- + +cache_stream::cache_stream(string const &path, + block_address block_size, + size_t cache_mem) + : block_size_(block_size), + nr_blocks_(get_nr_blocks(path, block_size)), + + // hack because cache uses LRU rather than MRU + cache_blocks_((cache_mem / block_size) / 2u), + fd_(open_file(path)), + v_(new bcache::noop_validator()), + cache_(new block_cache(fd_, block_size / 512, nr_blocks_, cache_mem)), + current_index_(0) { + load(0); + for (block_address i = 1; i < min(cache_blocks_, nr_blocks_); i++) + cache_->prefetch(i); +} + +block_address +cache_stream::nr_chunks() const +{ + return nr_blocks_; +} + +void +cache_stream::rewind() +{ + load(0); +} + +bool +cache_stream::advance(block_address count) +{ + if (current_index_ + count >= nr_blocks_) + return false; + + current_index_ += count; + + load(current_index_); + return true; +} + +block_address +cache_stream::index() const +{ + return current_index_; +} + +chunk const & +cache_stream::get() const +{ + return current_chunk_; +} + +void +cache_stream::load(block_address b) +{ + current_index_ = b; + current_block_ = cache_->get(current_index_, 0, v_); + + current_chunk_.offset_sectors_ = (b * block_size_) / 512; + current_chunk_.mem_.clear(); + current_chunk_.mem_.push_back(mem(static_cast(current_block_.get_data()), + static_cast(current_block_.get_data()) + block_size_)); + + if (current_index_ + cache_blocks_ < nr_blocks_) + cache_->prefetch(current_index_ + cache_blocks_); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/cache_stream.h b/thin-provisioning/cache_stream.h new file mode 100644 index 0000000..cfe6ca8 --- /dev/null +++ b/thin-provisioning/cache_stream.h @@ -0,0 +1,41 @@ +#ifndef THIN_PROVISIONING_CACHE_STREAM_H +#define THIN_PROVISIONING_CACHE_STREAM_H + +#include "thin-provisioning/chunk_stream.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + using namespace bcache; + + class cache_stream : public chunk_stream { + public: + cache_stream(std::string const &path, + block_address block_size, + size_t cache_mem); + + virtual block_address nr_chunks() const; + virtual void rewind(); + virtual bool advance(block_address count = 1ull); + virtual block_address index() const; + virtual chunk const &get() const; + + private: + void load(block_address b); + + block_address block_size_; + block_address nr_blocks_; + block_address cache_blocks_; + int fd_; + validator::ptr v_; + std::auto_ptr cache_; + + block_address current_index_; + block_cache::auto_block current_block_; + chunk current_chunk_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/chunk_stream.h b/thin-provisioning/chunk_stream.h new file mode 100644 index 0000000..d87ee9c --- /dev/null +++ b/thin-provisioning/chunk_stream.h @@ -0,0 +1,60 @@ +// Copyright (C) 2015 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#ifndef CHUNK_STREAM_H +#define CHUNK_STREAM_H + +#include "block-cache/block_cache.h" + +#include +#include + +//---------------------------------------------------------------- + +namespace thin_provisioning { + struct mem { + mem(uint8_t *b, uint8_t *e) + : begin(b), + end(e) { + } + + uint8_t *begin, *end; + }; + + struct chunk { + // FIXME: switch to bytes rather than sectors + // FIXME: add length too + uint64_t offset_sectors_; + std::deque mem_; + }; + + class chunk_stream { + public: + virtual ~chunk_stream() {} + + virtual bcache::block_address nr_chunks() const = 0; + virtual void rewind() = 0; + virtual bool advance(bcache::block_address count = 1ull) = 0; + virtual bcache::block_address index() const = 0; + virtual chunk const &get() const = 0; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/pool_stream.cc b/thin-provisioning/pool_stream.cc new file mode 100644 index 0000000..63b64ac --- /dev/null +++ b/thin-provisioning/pool_stream.cc @@ -0,0 +1,146 @@ +// Copyright (C) 2015 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include "thin-provisioning/pool_stream.h" +#include "persistent-data/data-structures/btree_damage_visitor.h" + +using namespace thin_provisioning; +using namespace persistent_data; + +//---------------------------------------------------------------- + +namespace { + class damage_visitor { + public: + virtual void visit(btree_path const &path, btree_detail::damage const &d) { + throw std::runtime_error("damage in mapping tree, please run thin_check"); + } + }; + + uint32_t const UNMAPPED = -1; +} + +//---------------------------------------------------------------- + +pool_stream::pool_stream(cache_stream &stream, + transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks) + : stream_(stream), + block_to_thin_(stream.nr_chunks(), UNMAPPED), + nr_mapped_(0) +{ + init_rmap(tm, sb, nr_blocks); +} + +block_address +pool_stream::nr_chunks() const +{ + return stream_.nr_chunks(); +} + +void +pool_stream::rewind() +{ + stream_.rewind(); +} + +bool +pool_stream::advance(block_address count) +{ + while (count--) + if (!advance_one()) + return false; + + return true; +} + +block_address +pool_stream::index() const +{ + return stream_.index(); +} + +chunk const & +pool_stream::get() const +{ + return stream_.get(); +} + + + +// FIXME: too big to return by value +vector +pool_stream::read_rmap(transaction_manager::ptr tm, + superblock_detail::superblock const &sb, + block_address nr_blocks) +{ + damage_visitor dv; + rmap_visitor rv; + + mapping_tree mtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::block_traits::ref_counter(tm->get_sm())); + + rv.add_data_region(rmap_visitor::region(0, nr_blocks)); + + btree_visit_values(mtree, rv, dv); + rv.complete(); + cerr << "rmap size: " << rv.get_rmap().size() << "\n"; + return rv.get_rmap(); +} + +void +pool_stream::init_rmap(transaction_manager::ptr tm, + superblock_detail::superblock const &sb, + block_address nr_blocks) +{ + cerr << "reading rmap..."; + vector rmap = read_rmap(tm, sb, nr_blocks); + cerr << "done\n"; + + vector::const_iterator it; + set thins; + for (it = rmap.begin(); it != rmap.end(); ++it) { + rmap_region const &r = *it; + for (block_address b = r.data_begin; b != r.data_end; b++) + if (block_to_thin_[b] == UNMAPPED) { + nr_mapped_++; + block_to_thin_[b] = r.thin_dev; + } + thins.insert(r.thin_dev); + } + + cerr << nr_mapped_ << " mapped blocks\n"; + cerr << "there are " << thins.size() << " thin devices\n"; +} + +bool +pool_stream::advance_one() +{ + block_address new_index = index() + 1; + + while (block_to_thin_[new_index] == UNMAPPED && + new_index < nr_chunks()) + new_index++; + + if (new_index >= nr_chunks()) + return false; + + return stream_.advance(new_index - index()); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/pool_stream.h b/thin-provisioning/pool_stream.h new file mode 100644 index 0000000..1fa5c51 --- /dev/null +++ b/thin-provisioning/pool_stream.h @@ -0,0 +1,60 @@ +// Copyright (C) 2015 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#ifndef POOL_STREAM_H +#define POOL_STREAM_H + +#include "thin-provisioning/cache_stream.h" +#include "thin-provisioning/rmap_visitor.h" +#include "thin-provisioning/superblock.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + class pool_stream : public chunk_stream { + public: + pool_stream(cache_stream &stream, + transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks); + + block_address nr_chunks() const; + void rewind(); + bool advance(block_address count = 1ull); + block_address index() const; + chunk const &get() const; + + private: + typedef rmap_visitor::region region; + typedef rmap_visitor::rmap_region rmap_region; + + // FIXME: too big to return by value + vector read_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks); + void init_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, + block_address nr_blocks); + bool advance_one(); + + cache_stream &stream_; + vector block_to_thin_; + block_address nr_mapped_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 96d1a23..261b971 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -29,11 +29,13 @@ #include "persistent-data/file_utils.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/space-maps/disk.h" +#include "thin-provisioning/cache_stream.h" +#include "thin-provisioning/pool_stream.h" #include "thin-provisioning/commands.h" #include "thin-provisioning/device_tree.h" #include "thin-provisioning/mapping_tree.h" -#include "thin-provisioning/superblock.h" #include "thin-provisioning/rmap_visitor.h" +#include "thin-provisioning/superblock.h" #include #include @@ -55,15 +57,6 @@ namespace { return (n % f) == 0; } - int open_file(string const &path) { - int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT | O_EXCL, 0666); - if (fd < 0) - syscall_failed("open", - "Note: you cannot run this tool with these options on live metadata."); - - return fd; - } - block_manager<>::ptr open_bm(string const &path) { block_address nr_blocks = get_nr_blocks(path); @@ -94,218 +87,6 @@ namespace { //-------------------------------- - // Once we start using variable sized blocks we will find we want - // to examine data that crosses cache block boundaries. So a block - // to be examined can be composed of multiple chunks of memory. - - struct mem { - mem(uint8_t *b, uint8_t *e) - : begin(b), - end(e) { - } - - uint8_t *begin, *end; - }; - - struct chunk { - // FIXME: switch to bytes rather than sectors - // FIXME: add length too - sector_t offset_sectors_; - deque mem_; - }; - - class chunk_stream { - public: - virtual ~chunk_stream() {} - - virtual block_address nr_chunks() const = 0; - virtual void rewind() = 0; - virtual bool advance(block_address count = 1ull) = 0; - virtual block_address index() const = 0; - virtual chunk const &get() const = 0; - }; - - class cache_stream : public chunk_stream { - public: - cache_stream(string const &path, - block_address block_size, - size_t cache_mem) - : block_size_(block_size), - nr_blocks_(get_nr_blocks(path, block_size)), - - // hack because cache uses LRU rather than MRU - cache_blocks_((cache_mem / block_size) / 2u), - fd_(open_file(path)), - v_(new bcache::noop_validator()), - cache_(new block_cache(fd_, block_size / 512, nr_blocks_, cache_mem)), - current_index_(0) { - load(0); - for (block_address i = 1; i < min(cache_blocks_, nr_blocks_); i++) - cache_->prefetch(i); - } - - virtual block_address nr_chunks() const { - return nr_blocks_; - } - - virtual void rewind() { - load(0); - } - - virtual bool advance(block_address count = 1ull) { - if (current_index_ + count >= nr_blocks_) - return false; - - current_index_ += count; - - load(current_index_); - return true; - } - - virtual block_address index() const { - return current_index_; - } - - virtual chunk const &get() const { - return current_chunk_; - } - - private: - void load(block_address b) { - current_index_ = b; - current_block_ = cache_->get(current_index_, 0, v_); - - current_chunk_.offset_sectors_ = (b * block_size_) / 512; - current_chunk_.mem_.clear(); - current_chunk_.mem_.push_back(mem(static_cast(current_block_.get_data()), - static_cast(current_block_.get_data()) + block_size_)); - - if (current_index_ + cache_blocks_ < nr_blocks_) - cache_->prefetch(current_index_ + cache_blocks_); - } - - block_address block_size_; - block_address nr_blocks_; - block_address cache_blocks_; - int fd_; - validator::ptr v_; - auto_ptr cache_; - - block_address current_index_; - block_cache::auto_block current_block_; - chunk current_chunk_; - }; - - //-------------------------------- - - typedef rmap_visitor::region region; - typedef rmap_visitor::rmap_region rmap_region; - - uint32_t const UNMAPPED = -1; - - class pool_stream : public chunk_stream { - public: - - pool_stream(cache_stream &stream, - transaction_manager::ptr tm, superblock_detail::superblock const &sb, - block_address nr_blocks) - : stream_(stream), - block_to_thin_(stream.nr_chunks(), UNMAPPED), - nr_mapped_(0) { - init_rmap(tm, sb, nr_blocks); - } - - block_address nr_chunks() const { - return stream_.nr_chunks(); - } - - void rewind() { - stream_.rewind(); - } - - bool advance(block_address count = 1ull) { - while (count--) - if (!advance_one()) - return false; - - return true; - } - - block_address index() const { - return stream_.index(); - } - - chunk const &get() const { - return stream_.get(); - } - - private: - class damage_visitor { - public: - virtual void visit(btree_path const &path, btree_detail::damage const &d) { - throw std::runtime_error("damage in mapping tree, please run thin_check"); - } - }; - - // FIXME: too big to return by value - vector read_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, - block_address nr_blocks) { - damage_visitor dv; - rmap_visitor rv; - - mapping_tree mtree(*tm, sb.data_mapping_root_, - mapping_tree_detail::block_traits::ref_counter(tm->get_sm())); - - rv.add_data_region(rmap_visitor::region(0, nr_blocks)); - - btree_visit_values(mtree, rv, dv); - rv.complete(); - cerr << "rmap size: " << rv.get_rmap().size() << "\n"; - return rv.get_rmap(); - } - - void init_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, - block_address nr_blocks) { - cerr << "reading rmap..."; - vector rmap = read_rmap(tm, sb, nr_blocks); - cerr << "done\n"; - - vector::const_iterator it; - set thins; - for (it = rmap.begin(); it != rmap.end(); ++it) { - rmap_region const &r = *it; - for (block_address b = r.data_begin; b != r.data_end; b++) - if (block_to_thin_[b] == UNMAPPED) { - nr_mapped_++; - block_to_thin_[b] = r.thin_dev; - } - thins.insert(r.thin_dev); - } - - cerr << nr_mapped_ << " mapped blocks\n"; - cerr << "there are " << thins.size() << " thin devices\n"; - } - - bool advance_one() { - block_address new_index = index() + 1; - - while (block_to_thin_[new_index] == UNMAPPED && - new_index < nr_chunks()) - new_index++; - - if (new_index >= nr_chunks()) - return false; - - return stream_.advance(new_index - index()); - } - - cache_stream &stream_; - vector block_to_thin_; - block_address nr_mapped_; - }; - - //-------------------------------- - struct flags { flags() : cache_mem(64 * 1024 * 1024) { @@ -434,6 +215,7 @@ namespace { return 0; } +#if 0 int show_dups_linear(flags const &fs) { if (!fs.block_size) // FIXME: this check should be moved to the switch parsing @@ -481,13 +263,14 @@ namespace { return 0; } +#endif int show_dups(flags const &fs) { if (fs.metadata_dev) return show_dups_pool(fs); else { cerr << "No metadata device provided, so treating data device as a linear device\n"; - return show_dups_linear(fs); + //return show_dups_linear(fs); } } From 10f93be8b1dc3062307be5ca5454d908c811b802 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 25 Aug 2015 08:22:16 +0100 Subject: [PATCH 014/118] [thin_show_dups] put linear branch back in --- thin-provisioning/pool_stream.cc | 2 +- thin-provisioning/thin_show_duplicates.cc | 38 ++++++----------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/thin-provisioning/pool_stream.cc b/thin-provisioning/pool_stream.cc index 63b64ac..0191698 100644 --- a/thin-provisioning/pool_stream.cc +++ b/thin-provisioning/pool_stream.cc @@ -50,7 +50,7 @@ pool_stream::pool_stream(cache_stream &stream, block_address pool_stream::nr_chunks() const { - return stream_.nr_chunks(); + return nr_mapped_; } void diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 261b971..62a4f77 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -210,12 +210,11 @@ namespace { pbar->update_percent(100); cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; -// cout << (detector.get_total_duplicates() * 100) / nr_mapped_ << "% duplicates\n"; + cout << (detector.get_total_duplicates() * 100) / pstream.nr_chunks() << "% duplicates\n"; return 0; } -#if 0 int show_dups_linear(flags const &fs) { if (!fs.block_size) // FIXME: this check should be moved to the switch parsing @@ -223,39 +222,21 @@ namespace { cerr << "path = " << fs.data_dev << "\n"; cerr << "block size = " << fs.block_size << "\n"; + block_address block_size = *fs.block_size * 512; block_address nr_blocks = get_nr_blocks(fs.data_dev, *fs.block_size); cerr << "nr_blocks = " << nr_blocks << "\n"; - // The cache uses a LRU eviction policy, which plays badly - // with a sequential read. So we can't prefetch all the - // blocks. - - // FIXME: add MRU policy to cache - unsigned cache_blocks = (fs.cache_mem / *fs.block_size) / 2; - int fd = open_file(fs.data_dev); - sector_t block_sectors = *fs.block_size / 512; - block_cache cache(fd, block_sectors, nr_blocks, fs.cache_mem); - validator::ptr v(new bcache::noop_validator()); + cache_stream stream(fs.data_dev, block_size, fs.cache_mem); duplicate_detector detector(*fs.block_size, nr_blocks); - // warm up the cache - for (block_address i = 0; i < cache_blocks; i++) - cache.prefetch(i); - auto_ptr pbar = create_progress_bar("Examining data"); + do { + chunk const &c = stream.get(); + detector.examine(c); + pbar->update_percent((stream.index() * 100) / stream.nr_chunks()); - for (block_address i = 0; i < nr_blocks; i++) { - block_cache::block &b = cache.get(i, 0, v); - block_address prefetch = i + cache_blocks; - if (prefetch < nr_blocks) - cache.prefetch(prefetch); - - detector.examine(b); - b.put(); - - pbar->update_percent(i * 100 / nr_blocks); - } + } while (stream.advance()); pbar->update_percent(100); cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; @@ -263,14 +244,13 @@ namespace { return 0; } -#endif int show_dups(flags const &fs) { if (fs.metadata_dev) return show_dups_pool(fs); else { cerr << "No metadata device provided, so treating data device as a linear device\n"; - //return show_dups_linear(fs); + return show_dups_linear(fs); } } From 6dd6fcb4cd24a0b19ed3584f7a684e33b84d6002 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 25 Aug 2015 08:38:01 +0100 Subject: [PATCH 015/118] [thin_show_dups] fix bug calculating block size for linear volumes --- thin-provisioning/thin_show_duplicates.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 62a4f77..6c07edc 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -220,15 +220,15 @@ namespace { // FIXME: this check should be moved to the switch parsing throw runtime_error("--block-sectors or --metadata-dev must be supplied"); - cerr << "path = " << fs.data_dev << "\n"; - cerr << "block size = " << fs.block_size << "\n"; - block_address block_size = *fs.block_size * 512; + block_address block_size = *fs.block_size; block_address nr_blocks = get_nr_blocks(fs.data_dev, *fs.block_size); + + cerr << "path = " << fs.data_dev << "\n"; cerr << "nr_blocks = " << nr_blocks << "\n"; + cerr << "block size = " << block_size << "\n"; cache_stream stream(fs.data_dev, block_size, fs.cache_mem); - - duplicate_detector detector(*fs.block_size, nr_blocks); + duplicate_detector detector(block_size, nr_blocks); auto_ptr pbar = create_progress_bar("Examining data"); do { From d44a817c60b4dbf05d1a6ae2a3b1c91bbef8b974 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 25 Aug 2015 09:14:40 +0100 Subject: [PATCH 016/118] [thin_show_dups] Track zero blocks --- thin-provisioning/thin_show_duplicates.cc | 94 ++++++++++++++--------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 6c07edc..7819ca1 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -106,47 +106,44 @@ namespace { public: duplicate_counter(block_address nr_blocks) : counts_(nr_blocks), - total_dups_(0) { + non_zero_dups_(0), + zero_dups_(0) { } void add_duplicate(block_address b1, block_address b2) { - total_dups_++; + non_zero_dups_++; counts_[b1]++; } + void add_zero_duplicate(block_address b) { + zero_dups_++; + } + block_address get_total() const { - return total_dups_; + return non_zero_dups_ + zero_dups_; + } + + block_address get_non_zeroes() const { + return non_zero_dups_; + } + + block_address get_zeroes() const { + return zero_dups_; } private: vector counts_; - block_address total_dups_; + block_address non_zero_dups_; + block_address zero_dups_; }; class duplicate_detector { public: duplicate_detector(unsigned block_size, block_address nr_blocks) : block_size_(block_size), - results_(nr_blocks) { - } - - // FIXME: remove - void examine(block_cache::block const &b) { - digestor_.reset(); - digestor_.process_bytes(b.get_data(), block_size_); - unsigned int digest[5]; - digestor_.get_digest(digest); - - // hack - vector v(5); - for (unsigned i = 0; i < 5; i++) - v[i] = digest[i]; - - fingerprint_map::const_iterator it = fm_.find(v); - if (it != fm_.end()) { - results_.add_duplicate(it->second, b.get_index()); - } else - fm_.insert(make_pair(v, b.get_index())); + results_(nr_blocks), + zero_fingerprint_(5, 0ull) { + calc_zero_fingerprint(); } void examine(chunk const &c) { @@ -163,16 +160,37 @@ namespace { for (unsigned i = 0; i < 5; i++) v[i] = digest[i]; - fingerprint_map::const_iterator it = fm_.find(v); block_address index = (c.offset_sectors_ * 512) / block_size_; - if (it != fm_.end()) { - results_.add_duplicate(it->second, index); - } else - fm_.insert(make_pair(v, index)); + + if (v == zero_fingerprint_) + results_.add_zero_duplicate(index); + + else { + fingerprint_map::const_iterator it = fm_.find(v); + if (it != fm_.end()) { + results_.add_duplicate(it->second, index); + } else + fm_.insert(make_pair(v, index)); + } } - block_address get_total_duplicates() const { - return results_.get_total(); + duplicate_counter const &get_results() const { + return results_; + } + + void calc_zero_fingerprint() { + auto_ptr bytes(new uint8_t[block_size_]); + memset(bytes.get(), 0, block_size_); + + digestor_.reset(); + digestor_.process_bytes(bytes.get(), block_size_); + + unsigned int digest[5]; + digestor_.get_digest(digest); + + // hack + for (unsigned i = 0; i < 5; i++) + zero_fingerprint_[i] = digest[i]; } private: @@ -182,6 +200,8 @@ namespace { boost::uuids::detail::sha1 digestor_; fingerprint_map fm_; duplicate_counter results_; + + vector zero_fingerprint_; }; int show_dups_pool(flags const &fs) { @@ -209,8 +229,8 @@ namespace { } while (pstream.advance()); pbar->update_percent(100); - cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; - cout << (detector.get_total_duplicates() * 100) / pstream.nr_chunks() << "% duplicates\n"; + cout << "\n\ntotal dups: " << detector.get_results().get_total() << endl; + cout << (detector.get_results().get_total() * 100) / pstream.nr_chunks() << "% duplicates\n"; return 0; } @@ -239,8 +259,12 @@ namespace { } while (stream.advance()); pbar->update_percent(100); - cout << "\n\ntotal dups: " << detector.get_total_duplicates() << endl; - cout << (detector.get_total_duplicates() * 100) / nr_blocks << "% duplicates\n"; + cout << "\n\ntotal dups: " << detector.get_results().get_total() << endl; + cout << (detector.get_results().get_total() * 100) / nr_blocks << "% duplicates\n"; + + duplicate_counter r = detector.get_results(); + cout << "\n\nchunks\tnon zero dups\tzero dups\n" + << nr_blocks << "\t" << r.get_non_zeroes() << "\t" << r.get_zeroes() << "\n"; return 0; } From 750ce0f47b616907b110fbe8afc1508efd42c2ed Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 3 Sep 2015 13:02:29 +0100 Subject: [PATCH 017/118] [thin_show_dups] variable_chunk_stream --- Makefile.in | 3 + base/rolling_hash.cc | 148 ++++++++++++++++++++ base/rolling_hash.h | 61 ++++++++ thin-provisioning/cache_stream.cc | 53 ++++--- thin-provisioning/cache_stream.h | 23 +++- thin-provisioning/chunk_stream.cc | 23 ++++ thin-provisioning/chunk_stream.h | 15 +- thin-provisioning/pool_stream.cc | 18 ++- thin-provisioning/pool_stream.h | 10 +- thin-provisioning/thin_show_duplicates.cc | 111 +++++++-------- thin-provisioning/variable_chunk_stream.cc | 152 ++++++++++++++++++++ thin-provisioning/variable_chunk_stream.h | 42 ++++++ unit-tests/Makefile.in | 1 + unit-tests/rolling_hash_t.cc | 153 +++++++++++++++++++++ 14 files changed, 709 insertions(+), 104 deletions(-) create mode 100644 base/rolling_hash.cc create mode 100644 base/rolling_hash.h create mode 100644 thin-provisioning/chunk_stream.cc create mode 100644 thin-provisioning/variable_chunk_stream.cc create mode 100644 thin-provisioning/variable_chunk_stream.h create mode 100644 unit-tests/rolling_hash_t.cc diff --git a/Makefile.in b/Makefile.in index 0dd6c99..d09c873 100644 --- a/Makefile.in +++ b/Makefile.in @@ -31,6 +31,7 @@ SOURCE=\ base/error_state.cc \ base/error_string.cc \ base/progress_monitor.cc \ + base/rolling_hash.cc \ base/xml_utils.cc \ block-cache/block_cache.cc \ caching/cache_check.cc \ @@ -72,6 +73,7 @@ SOURCE=\ persistent-data/transaction_manager.cc \ persistent-data/validators.cc \ thin-provisioning/cache_stream.cc \ + thin-provisioning/chunk_stream.cc \ thin-provisioning/device_tree.cc \ thin-provisioning/human_readable_format.cc \ thin-provisioning/mapping_tree.cc \ @@ -92,6 +94,7 @@ SOURCE=\ thin-provisioning/thin_rmap.cc \ thin-provisioning/thin_show_duplicates.cc \ thin-provisioning/thin_trim.cc \ + thin-provisioning/variable_chunk_stream.cc \ thin-provisioning/xml_format.cc CC:=@CC@ diff --git a/base/rolling_hash.cc b/base/rolling_hash.cc new file mode 100644 index 0000000..1ea362f --- /dev/null +++ b/base/rolling_hash.cc @@ -0,0 +1,148 @@ +#include "base/rolling_hash.h" + +using namespace base; +using namespace boost; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + uint32_t MULTIPLIER = 4294967291UL; + uint32_t SEED = 123; +} + +rolling_hash::rolling_hash(unsigned window_size) + : a_(MULTIPLIER), + a_to_k_minus_1_(a_), + window_size_(window_size) { + + for (unsigned i = 1; i < window_size_ - 1; i++) + a_to_k_minus_1_ *= a_; + + reset(); +} + +void +rolling_hash::reset() +{ + // prime with zeroes + chars_.clear(); + + hash_ = 0; + for (unsigned i = 0; i < window_size_; i++) { + hash_ = (hash_ * a_) + SEED; + chars_.push_back(0); + } +} + +uint32_t +rolling_hash::step(uint8_t byte) +{ + update_hash(byte); + return hash_; +} + +uint32_t +rolling_hash::get_hash() const +{ + return hash_; +} + +void +rolling_hash::update_hash(uint8_t byte) +{ + hash_ -= a_to_k_minus_1_ * (chars_.front() + SEED); + chars_.pop_front(); + chars_.push_back(byte); + hash_ = (hash_ * a_) + byte + SEED; +} + +//-------------------------------- + +content_based_hash::content_based_hash(unsigned window_size) + : rhash_(window_size), + + // FIXME: hard coded values + backup_div_((window_size / 4) - 1), + div_((window_size / 2) - 1), + min_len_(window_size / 8), + max_len_(window_size), + len_(0) +{ +} + +void +content_based_hash::reset() +{ + len_ = 0; + backup_break_.reset(); + rhash_.reset(); +} + +optional +content_based_hash::step(uint8_t byte) +{ +#if 0 + optional r; + + rhash_.step(byte); + len_++; + + if (len_ < min_len_) + return r; + + if (hit_break(backup_div_)) + backup_break_ = len_; + + if (hit_break(div_)) { + // found a break + r = len_; + len_ = 0; + backup_break_.reset(); + + } else if (len_ >= max_len_) { + // too big, is there a backup? + if (backup_break_) { + len_ -= *backup_break_; + r = backup_break_; + backup_break_.reset(); + + } else { + r = len_; + len_ = 0; + } + } + + return r; +#else + optional r; + + rhash_.step(byte); + len_++; + + if (len_ < min_len_) + return r; + + if (hit_break(div_)) { + // found a break + r = len_; + len_ = 0; + backup_break_.reset(); + + } else if (len_ >= max_len_) { + r = len_; + len_ = 0; + } + + return r; +#endif +} + +bool +content_based_hash::hit_break(uint32_t mask) const +{ + uint32_t h = rhash_.get_hash() >> 8; + return !(h & mask); +} + +//---------------------------------------------------------------- diff --git a/base/rolling_hash.h b/base/rolling_hash.h new file mode 100644 index 0000000..d44012a --- /dev/null +++ b/base/rolling_hash.h @@ -0,0 +1,61 @@ +#ifndef BASE_ROLLING_HASH_H +#define BASE_ROLLING_HASH_H + +#include +#include +#include + +//---------------------------------------------------------------- + +namespace base { + class rolling_hash { + public: + rolling_hash(unsigned window_size); + + void reset(); + + // Returns the current hash + uint32_t step(uint8_t byte); + + uint32_t get_hash() const; + + private: + void update_hash(uint8_t byte); + + uint32_t a_; + uint32_t a_to_k_minus_1_; + + // FIXME: use a ring buffer + std::list chars_; + + uint32_t hash_; + uint32_t window_size_; + }; + + class content_based_hash { + public: + content_based_hash(unsigned window_size); + void reset(); + + // Returns a break point relative to the last reset/break. + boost::optional step(uint8_t byte); + + private: + bool hit_break(uint32_t div) const; + + rolling_hash rhash_; + + uint32_t backup_div_; + uint32_t div_; + + unsigned min_len_; + unsigned max_len_; + + unsigned len_; + boost::optional backup_break_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/cache_stream.cc b/thin-provisioning/cache_stream.cc index 956fc4e..8fbcc72 100644 --- a/thin-provisioning/cache_stream.cc +++ b/thin-provisioning/cache_stream.cc @@ -32,9 +32,8 @@ cache_stream::cache_stream(string const &path, v_(new bcache::noop_validator()), cache_(new block_cache(fd_, block_size / 512, nr_blocks_, cache_mem)), current_index_(0) { - load(0); - for (block_address i = 1; i < min(cache_blocks_, nr_blocks_); i++) - cache_->prefetch(i); + + rewind(); } block_address @@ -46,19 +45,27 @@ cache_stream::nr_chunks() const void cache_stream::rewind() { - load(0); + current_index_ = 0; + + for (block_address i = 1; i < min(cache_blocks_, nr_blocks_); i++) + cache_->prefetch(i); } bool -cache_stream::advance(block_address count) +cache_stream::next(block_address count) { - if (current_index_ + count >= nr_blocks_) - return false; + current_index_ = min(current_index_ + count, nr_blocks_); - current_index_ += count; + if (current_index_ + cache_blocks_ < nr_blocks_) + cache_->prefetch(current_index_ + cache_blocks_); - load(current_index_); - return true; + return !eof(); +} + +bool +cache_stream::eof() const +{ + return current_index_ >= nr_blocks_; } block_address @@ -68,24 +75,26 @@ cache_stream::index() const } chunk const & -cache_stream::get() const +cache_stream::get() { - return current_chunk_; + chunk_wrapper *w = new chunk_wrapper(*this); + return w->c_; } void -cache_stream::load(block_address b) +cache_stream::put(chunk const &c) { - current_index_ = b; - current_block_ = cache_->get(current_index_, 0, v_); + chunk_wrapper *w = container_of(const_cast(&c), chunk_wrapper, c_); + delete w; +} - current_chunk_.offset_sectors_ = (b * block_size_) / 512; - current_chunk_.mem_.clear(); - current_chunk_.mem_.push_back(mem(static_cast(current_block_.get_data()), - static_cast(current_block_.get_data()) + block_size_)); - - if (current_index_ + cache_blocks_ < nr_blocks_) - cache_->prefetch(current_index_ + cache_blocks_); +cache_stream::chunk_wrapper::chunk_wrapper(cache_stream &parent) + : block_(parent.cache_->get(parent.current_index_, 0, parent.v_)) +{ + c_.offset_ = parent.current_index_ * parent.block_size_; + c_.len_ = parent.block_size_; + c_.mem_.push_back(mem(static_cast(block_.get_data()), + static_cast(block_.get_data()) + parent.block_size_)); } //---------------------------------------------------------------- diff --git a/thin-provisioning/cache_stream.h b/thin-provisioning/cache_stream.h index cfe6ca8..65c81b1 100644 --- a/thin-provisioning/cache_stream.h +++ b/thin-provisioning/cache_stream.h @@ -14,25 +14,36 @@ namespace thin_provisioning { block_address block_size, size_t cache_mem); - virtual block_address nr_chunks() const; + block_address nr_chunks() const; + virtual void rewind(); - virtual bool advance(block_address count = 1ull); virtual block_address index() const; - virtual chunk const &get() const; + + virtual bool next(block_address count = 1ull); + virtual bool eof() const; + + virtual chunk const &get(); + virtual void put(chunk const &c); private: - void load(block_address b); + struct chunk_wrapper { + chunk_wrapper(cache_stream &parent); + + block_cache::auto_block block_; + chunk c_; + }; + + friend class chunk_wrapper; block_address block_size_; block_address nr_blocks_; block_address cache_blocks_; + int fd_; validator::ptr v_; std::auto_ptr cache_; block_address current_index_; - block_cache::auto_block current_block_; - chunk current_chunk_; }; } diff --git a/thin-provisioning/chunk_stream.cc b/thin-provisioning/chunk_stream.cc new file mode 100644 index 0000000..4ac99ff --- /dev/null +++ b/thin-provisioning/chunk_stream.cc @@ -0,0 +1,23 @@ +#include "thin-provisioning/chunk_stream.h" + +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +uint8_t +chunk::operator[](uint64_t n) const +{ + std::deque::const_iterator it; + for (it = mem_.begin(); it != mem_.end(); it++) { + uint64_t mem_len = it->end - it->begin; + if (n > mem_len) + n -= mem_len; + else + return it->begin[n]; + } + + throw runtime_error("chunk out of bounds"); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/chunk_stream.h b/thin-provisioning/chunk_stream.h index d87ee9c..4e1ea96 100644 --- a/thin-provisioning/chunk_stream.h +++ b/thin-provisioning/chunk_stream.h @@ -37,21 +37,24 @@ namespace thin_provisioning { }; struct chunk { - // FIXME: switch to bytes rather than sectors - // FIXME: add length too - uint64_t offset_sectors_; + uint64_t offset_, len_; std::deque mem_; + + uint8_t operator[](uint64_t n) const; }; class chunk_stream { public: virtual ~chunk_stream() {} - virtual bcache::block_address nr_chunks() const = 0; virtual void rewind() = 0; - virtual bool advance(bcache::block_address count = 1ull) = 0; virtual bcache::block_address index() const = 0; - virtual chunk const &get() const = 0; + + virtual bool next(bcache::block_address count = 1ull) = 0; + virtual bool eof() const = 0; + + virtual chunk const &get() = 0; + virtual void put(chunk const &c) = 0; }; } diff --git a/thin-provisioning/pool_stream.cc b/thin-provisioning/pool_stream.cc index 0191698..21964f9 100644 --- a/thin-provisioning/pool_stream.cc +++ b/thin-provisioning/pool_stream.cc @@ -60,7 +60,7 @@ pool_stream::rewind() } bool -pool_stream::advance(block_address count) +pool_stream::next(block_address count) { while (count--) if (!advance_one()) @@ -69,6 +69,12 @@ pool_stream::advance(block_address count) return true; } +bool +pool_stream::eof() const +{ + return stream_.eof(); +} + block_address pool_stream::index() const { @@ -76,12 +82,16 @@ pool_stream::index() const } chunk const & -pool_stream::get() const +pool_stream::get() { return stream_.get(); } - +void +pool_stream::put(chunk const &c) +{ + stream_.put(c); +} // FIXME: too big to return by value vector @@ -140,7 +150,7 @@ pool_stream::advance_one() if (new_index >= nr_chunks()) return false; - return stream_.advance(new_index - index()); + return stream_.next(new_index - index()); } //---------------------------------------------------------------- diff --git a/thin-provisioning/pool_stream.h b/thin-provisioning/pool_stream.h index 1fa5c51..71576ed 100644 --- a/thin-provisioning/pool_stream.h +++ b/thin-provisioning/pool_stream.h @@ -34,16 +34,20 @@ namespace thin_provisioning { block_address nr_chunks() const; void rewind(); - bool advance(block_address count = 1ull); + bool next(block_address count = 1ull); + bool eof() const; block_address index() const; - chunk const &get() const; + + chunk const &get(); + void put(chunk const &c); private: typedef rmap_visitor::region region; typedef rmap_visitor::rmap_region rmap_region; // FIXME: too big to return by value - vector read_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, + vector read_rmap(transaction_manager::ptr tm, + superblock_detail::superblock const &sb, block_address nr_blocks); void init_rmap(transaction_manager::ptr tm, superblock_detail::superblock const &sb, block_address nr_blocks); diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 7819ca1..dafcd34 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -36,6 +36,7 @@ #include "thin-provisioning/mapping_tree.h" #include "thin-provisioning/rmap_visitor.h" #include "thin-provisioning/superblock.h" +#include "thin-provisioning/variable_chunk_stream.h" #include #include @@ -98,25 +99,21 @@ namespace { unsigned cache_mem; }; - // FIXME: introduce abstraction for a stream of segments - using namespace mapping_tree_detail; class duplicate_counter { public: - duplicate_counter(block_address nr_blocks) - : counts_(nr_blocks), - non_zero_dups_(0), + duplicate_counter() + : non_zero_dups_(0), zero_dups_(0) { } - void add_duplicate(block_address b1, block_address b2) { - non_zero_dups_++; - counts_[b1]++; + void add_duplicate(block_address len) { + non_zero_dups_ += len; } - void add_zero_duplicate(block_address b) { - zero_dups_++; + void add_zero_duplicate(block_address len) { + zero_dups_ += len; } block_address get_total() const { @@ -132,45 +129,35 @@ namespace { } private: - vector counts_; block_address non_zero_dups_; block_address zero_dups_; }; class duplicate_detector { public: - duplicate_detector(unsigned block_size, block_address nr_blocks) - : block_size_(block_size), - results_(nr_blocks), - zero_fingerprint_(5, 0ull) { - calc_zero_fingerprint(); - } - void examine(chunk const &c) { - digestor_.reset(); - - for (deque::const_iterator it = c.mem_.begin(); it != c.mem_.end(); it++) - digestor_.process_bytes(it->begin, it->end - it->begin); - - unsigned int digest[5]; - digestor_.get_digest(digest); - - // hack - vector v(5); - for (unsigned i = 0; i < 5; i++) - v[i] = digest[i]; - - block_address index = (c.offset_sectors_ * 512) / block_size_; - - if (v == zero_fingerprint_) - results_.add_zero_duplicate(index); + if (all_zeroes(c)) + results_.add_zero_duplicate(c.len_); else { + digestor_.reset(); + + for (deque::const_iterator it = c.mem_.begin(); it != c.mem_.end(); it++) + digestor_.process_bytes(it->begin, it->end - it->begin); + + unsigned int digest[5]; + digestor_.get_digest(digest); + + // hack + vector v(5); + for (unsigned i = 0; i < 5; i++) + v[i] = digest[i]; + fingerprint_map::const_iterator it = fm_.find(v); if (it != fm_.end()) { - results_.add_duplicate(it->second, index); + results_.add_duplicate(c.len_); } else - fm_.insert(make_pair(v, index)); + fm_.insert(make_pair(v, c.offset_)); } } @@ -178,30 +165,24 @@ namespace { return results_; } - void calc_zero_fingerprint() { - auto_ptr bytes(new uint8_t[block_size_]); - memset(bytes.get(), 0, block_size_); + private: + bool all_zeroes(chunk const &c) const { + for (deque::const_iterator it = c.mem_.begin(); it != c.mem_.end(); it++) { + for (uint8_t *ptr = it->begin; ptr != it->end; ptr++) { + if (*ptr != 0) + return false; + } + } - digestor_.reset(); - digestor_.process_bytes(bytes.get(), block_size_); - - unsigned int digest[5]; - digestor_.get_digest(digest); - - // hack - for (unsigned i = 0; i < 5; i++) - zero_fingerprint_[i] = digest[i]; + return true; } - private: typedef map, block_address> fingerprint_map; unsigned block_size_; boost::uuids::detail::sha1 digestor_; fingerprint_map fm_; duplicate_counter results_; - - vector zero_fingerprint_; }; int show_dups_pool(flags const &fs) { @@ -218,15 +199,16 @@ namespace { cache_stream stream(fs.data_dev, block_size, fs.cache_mem); pool_stream pstream(stream, tm, sb, nr_blocks); - duplicate_detector detector(block_size, nr_blocks); + duplicate_detector detector; auto_ptr pbar = create_progress_bar("Examining data"); do { chunk const &c = pstream.get(); detector.examine(c); + pstream.put(c); pbar->update_percent((pstream.index() * 100) / pstream.nr_chunks()); - } while (pstream.advance()); + } while (pstream.next()); pbar->update_percent(100); cout << "\n\ntotal dups: " << detector.get_results().get_total() << endl; @@ -247,24 +229,27 @@ namespace { cerr << "nr_blocks = " << nr_blocks << "\n"; cerr << "block size = " << block_size << "\n"; - cache_stream stream(fs.data_dev, block_size, fs.cache_mem); - duplicate_detector detector(block_size, nr_blocks); + cache_stream low_level_stream(fs.data_dev, block_size, fs.cache_mem); + variable_chunk_stream stream(low_level_stream, 4096); + duplicate_detector detector; auto_ptr pbar = create_progress_bar("Examining data"); do { + // FIXME: use a wrapper class to automate the put() chunk const &c = stream.get(); detector.examine(c); - pbar->update_percent((stream.index() * 100) / stream.nr_chunks()); + stream.put(c); +// pbar->update_percent((stream.index() * 100) / stream.nr_chunks()); - } while (stream.advance()); + } while (stream.next()); pbar->update_percent(100); - cout << "\n\ntotal dups: " << detector.get_results().get_total() << endl; - cout << (detector.get_results().get_total() * 100) / nr_blocks << "% duplicates\n"; - duplicate_counter r = detector.get_results(); - cout << "\n\nchunks\tnon zero dups\tzero dups\n" - << nr_blocks << "\t" << r.get_non_zeroes() << "\t" << r.get_zeroes() << "\n"; + block_address meg = 1024 * 1024; + cout << "\n\n" + << (nr_blocks * block_size) / meg << "m examined, " + << r.get_non_zeroes() / meg << "m duplicates, " + << r.get_zeroes() / meg << "m zeroes\n"; return 0; } diff --git a/thin-provisioning/variable_chunk_stream.cc b/thin-provisioning/variable_chunk_stream.cc new file mode 100644 index 0000000..9a9d11e --- /dev/null +++ b/thin-provisioning/variable_chunk_stream.cc @@ -0,0 +1,152 @@ +#include "thin-provisioning/variable_chunk_stream.h" + +using namespace boost; +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +variable_chunk_stream::variable_chunk_stream(chunk_stream &stream, unsigned window_size) + : index_(0), + h_(window_size), + stream_(stream), + big_chunk_(0) { + next_big_chunk(); +} + +variable_chunk_stream::~variable_chunk_stream() +{ + put_big_chunk(); +} + +void +variable_chunk_stream::rewind() +{ + // FIXME: not complete + index_ = 0; + stream_.rewind(); + h_.reset(); +} + +bool +variable_chunk_stream::next(bcache::block_address count) +{ + while (count--) { + index_++; + advance_one(); + } + + return !eof(); +} + +bool +variable_chunk_stream::eof() const +{ + return stream_.eof(); +} + +bcache::block_address +variable_chunk_stream::index() const +{ + return index_; +} + +chunk const & +variable_chunk_stream::get() +{ + assert(big_chunk_); + + little_chunk_.len_ = little_e_ - little_b_; + little_chunk_.offset_ = big_chunk_->offset_ + little_chunk_.len_; + + little_chunk_.mem_.clear(); + little_chunk_.mem_.push_back(mem(little_b_, little_e_)); + + return little_chunk_; +} + +void +variable_chunk_stream::put(chunk const &c) +{ + // noop +} + +bool +variable_chunk_stream::next_big_chunk() +{ + put_big_chunk(); + + if (!stream_.next()) + return false; + + big_chunk_ = &stream_.get(); + little_b_ = little_e_ = big_chunk_->mem_.front().begin; + h_.reset(); + + return true; +} + +bool +variable_chunk_stream::advance_one() +{ + uint8_t *big_e; + + assert(big_chunk_); + + big_e = big_chunk_->mem_.front().end; + little_b_ = little_e_; + + if (little_b_ == big_e) { + if (next_big_chunk()) + big_e = big_chunk_->mem_.front().end; + else + return false; + } + + assert(little_e_ >= big_chunk_->mem_.front().begin); + assert(little_b_ >= big_chunk_->mem_.front().begin); +#if 1 + if (little_e_ > big_e) { + cerr << "before -- little_e_: " << (void *) little_e_ << ", big_e: " << (void *) big_e << "\n"; + } +#endif + assert(little_e_ <= big_e); + assert(little_b_ <= big_e); + + + while (little_e_ != big_e) { + optional maybe_break = h_.step(*little_e_); + + if (maybe_break) { + // The break is not neccessarily at the current + // byte. + little_e_ = little_b_ + *maybe_break; + break; + } + + little_e_++; + } + + assert(little_e_ >= big_chunk_->mem_.front().begin); + assert(little_b_ >= big_chunk_->mem_.front().begin); +#if 1 + if (little_e_ > big_e) { + cerr << "after -- little_e_: " << (void *) little_e_ << ", big_e: " << (void *) big_e << "\n"; + } +#endif + assert(little_e_ <= big_e); + assert(little_b_ <= big_e); + + return true; +} + +void +variable_chunk_stream::put_big_chunk() +{ + if (big_chunk_) + stream_.put(*big_chunk_); + + big_chunk_ = 0; +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/variable_chunk_stream.h b/thin-provisioning/variable_chunk_stream.h new file mode 100644 index 0000000..0327f1d --- /dev/null +++ b/thin-provisioning/variable_chunk_stream.h @@ -0,0 +1,42 @@ +#ifndef THIN_PROVISIONING_VARIABLE_CHUNK_STREAM_H +#define THIN_PROVISIONING_VARIABLE_CHUNK_STREAM_H + +#include "base/rolling_hash.h" +#include "thin-provisioning/chunk_stream.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + class variable_chunk_stream : public chunk_stream { + public: + // window_size must be a power of 2 + variable_chunk_stream(chunk_stream &stream, unsigned window_size); + ~variable_chunk_stream(); + + // FIXME: we don't know in advance how many chunks we will have + virtual void rewind(); + virtual bool next(bcache::block_address count = 1ull); + virtual bool eof() const; + virtual bcache::block_address index() const; + virtual chunk const &get(); + virtual void put(chunk const &c); + + private: + bool next_big_chunk(); + bool advance_one(); + void put_big_chunk(); + + bcache::block_address index_; + base::content_based_hash h_; + + chunk_stream &stream_; + chunk const *big_chunk_; + + uint8_t *little_b_, *little_e_; + chunk little_chunk_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/unit-tests/Makefile.in b/unit-tests/Makefile.in index 9307e5f..38f3d04 100644 --- a/unit-tests/Makefile.in +++ b/unit-tests/Makefile.in @@ -59,6 +59,7 @@ TEST_SOURCE=\ unit-tests/endian_t.cc \ unit-tests/error_state_t.cc \ unit-tests/rmap_visitor_t.cc \ + unit-tests/rolling_hash_t.cc \ unit-tests/run_set_t.cc \ unit-tests/space_map_t.cc \ unit-tests/span_iterator_t.cc \ diff --git a/unit-tests/rolling_hash_t.cc b/unit-tests/rolling_hash_t.cc new file mode 100644 index 0000000..c25b650 --- /dev/null +++ b/unit-tests/rolling_hash_t.cc @@ -0,0 +1,153 @@ +#include "gmock/gmock.h" + +#include "base/rolling_hash.h" + +using namespace base; +using namespace boost; +using namespace std; +using namespace testing; + +//---------------------------------------------------------------- + +namespace { + class RollingHashTests : public Test { + public: + RollingHashTests() + : window_size_(4096), + rhash_(window_size_) { + } + + typedef vector bytes; + bytes random_bytes(unsigned count) { + bytes v(count, 0); + + for (unsigned i = 0; i < count; i++) + v[i] = random_byte(); + + return v; + } + + uint8_t random_byte() const { + return random() % 256; + } + + void apply_bytes(bytes const &bs) { + for (unsigned i = 0; i < bs.size(); i++) + rhash_.step(bs[i]); + } + + unsigned window_size_; + rolling_hash rhash_; + }; + + class ContentBasedHashTests : public Test { + public: + ContentBasedHashTests() + : window_size_(8192), + h_(window_size_) { + } + + typedef vector bytes; + bytes random_bytes(unsigned count) { + bytes v(count, 0); + + for (unsigned i = 0; i < count; i++) + v[i] = random_byte(); + + return v; + } + + uint8_t random_byte() const { + return random() % 256; + } + + unsigned window_size_; + content_based_hash h_; + }; +} + +//---------------------------------------------------------------- + +TEST_F(RollingHashTests, ctr) +{ +} + +//-------------------------------- + +TEST_F(RollingHashTests, hash_changes) +{ + bytes bs = random_bytes(window_size_ * 100); + + uint32_t prev = rhash_.get_hash(); + for (unsigned i = 0; i < bs.size(); i++) { + rhash_.step(bs[i]); + ASSERT_NE(rhash_.get_hash(), prev); + prev = rhash_.get_hash(); + } +} + +TEST_F(RollingHashTests, hash_repeats) +{ + bytes bs = random_bytes(window_size_); + + apply_bytes(bs); + uint32_t h1 = rhash_.get_hash(); + apply_bytes(bs); + + ASSERT_EQ(rhash_.get_hash(), h1); +} + +TEST_F(RollingHashTests, reset_is_deterministic) +{ + uint8_t bytes[] = "lksdfuwerh,sdg"; + + for (unsigned i = 0; i < sizeof(bytes) - 1; i++) + rhash_.step(bytes[i]); + + uint32_t h1 = rhash_.get_hash(); + + rhash_.reset(); + + for (unsigned i = 0; i < sizeof(bytes) - 1; i++) + rhash_.step(bytes[i]); + + uint32_t h2 = rhash_.get_hash(); + + ASSERT_EQ(h1, h2); +} + +//---------------------------------------------------------------- + +TEST_F(ContentBasedHashTests, ctr) +{ +} + +TEST_F(ContentBasedHashTests, chunk_limits_respected) +{ + unsigned min = 100000, max = 0; + + bytes bs = random_bytes(1024 * 1024 * 100); + vector counts(window_size_, 0); + + for (unsigned i = 0; i < bs.size(); i++) { + optional b = h_.step(bs[i]); + if (b) { + counts[*b]++; + + if (*b < min) + min = *b; + + if (*b > max) + max = *b; + } + } + +#if 1 + for (unsigned i = 0; i < counts.size(); i++) + cerr << i << ": " << counts[i] << "\n"; + + cerr << "min: " << min << ", max: " << max << "\n"; +#endif +} + +//---------------------------------------------------------------- From 5d383c029334bb8d85dee5c81c40741c2975cf01 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 10:10:41 +0100 Subject: [PATCH 018/118] [thin_show_dups] get the backup break working in the rolling hash --- base/rolling_hash.cc | 23 ---------------------- thin-provisioning/variable_chunk_stream.cc | 20 +++++++------------ thin-provisioning/variable_chunk_stream.h | 2 +- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/base/rolling_hash.cc b/base/rolling_hash.cc index 1ea362f..9c6e1bf 100644 --- a/base/rolling_hash.cc +++ b/base/rolling_hash.cc @@ -82,7 +82,6 @@ content_based_hash::reset() optional content_based_hash::step(uint8_t byte) { -#if 0 optional r; rhash_.step(byte); @@ -114,28 +113,6 @@ content_based_hash::step(uint8_t byte) } return r; -#else - optional r; - - rhash_.step(byte); - len_++; - - if (len_ < min_len_) - return r; - - if (hit_break(div_)) { - // found a break - r = len_; - len_ = 0; - backup_break_.reset(); - - } else if (len_ >= max_len_) { - r = len_; - len_ = 0; - } - - return r; -#endif } bool diff --git a/thin-provisioning/variable_chunk_stream.cc b/thin-provisioning/variable_chunk_stream.cc index 9a9d11e..41c6c96 100644 --- a/thin-provisioning/variable_chunk_stream.cc +++ b/thin-provisioning/variable_chunk_stream.cc @@ -80,7 +80,7 @@ variable_chunk_stream::next_big_chunk() return false; big_chunk_ = &stream_.get(); - little_b_ = little_e_ = big_chunk_->mem_.front().begin; + little_b_ = little_e_ = last_hashed_ = big_chunk_->mem_.front().begin; h_.reset(); return true; @@ -95,6 +95,7 @@ variable_chunk_stream::advance_one() big_e = big_chunk_->mem_.front().end; little_b_ = little_e_; + little_e_ = last_hashed_; if (little_b_ == big_e) { if (next_big_chunk()) @@ -105,35 +106,28 @@ variable_chunk_stream::advance_one() assert(little_e_ >= big_chunk_->mem_.front().begin); assert(little_b_ >= big_chunk_->mem_.front().begin); -#if 1 - if (little_e_ > big_e) { - cerr << "before -- little_e_: " << (void *) little_e_ << ", big_e: " << (void *) big_e << "\n"; - } -#endif assert(little_e_ <= big_e); assert(little_b_ <= big_e); while (little_e_ != big_e) { optional maybe_break = h_.step(*little_e_); + little_e_++; if (maybe_break) { // The break is not neccessarily at the current // byte. + last_hashed_ = little_e_; little_e_ = little_b_ + *maybe_break; break; } - - little_e_++; } + if (little_e_ == big_e) + last_hashed_ = little_e_; + assert(little_e_ >= big_chunk_->mem_.front().begin); assert(little_b_ >= big_chunk_->mem_.front().begin); -#if 1 - if (little_e_ > big_e) { - cerr << "after -- little_e_: " << (void *) little_e_ << ", big_e: " << (void *) big_e << "\n"; - } -#endif assert(little_e_ <= big_e); assert(little_b_ <= big_e); diff --git a/thin-provisioning/variable_chunk_stream.h b/thin-provisioning/variable_chunk_stream.h index 0327f1d..f9c5ec7 100644 --- a/thin-provisioning/variable_chunk_stream.h +++ b/thin-provisioning/variable_chunk_stream.h @@ -32,7 +32,7 @@ namespace thin_provisioning { chunk_stream &stream_; chunk const *big_chunk_; - uint8_t *little_b_, *little_e_; + uint8_t *little_b_, *little_e_, *last_hashed_; chunk little_chunk_; }; } From 7633c5d7ae81b2b803ace3a9347f0fe510fa60ff Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 10:36:39 +0100 Subject: [PATCH 019/118] [thin_show_dups] get the progress bar working again --- thin-provisioning/thin_show_duplicates.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index dafcd34..9fc96c2 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -224,6 +224,7 @@ namespace { block_address block_size = *fs.block_size; block_address nr_blocks = get_nr_blocks(fs.data_dev, *fs.block_size); + block_address dev_size = nr_blocks * *fs.block_size; cerr << "path = " << fs.data_dev << "\n"; cerr << "nr_blocks = " << nr_blocks << "\n"; @@ -239,7 +240,8 @@ namespace { chunk const &c = stream.get(); detector.examine(c); stream.put(c); -// pbar->update_percent((stream.index() * 100) / stream.nr_chunks()); + + pbar->update_percent((c.offset_ * 100) / dev_size); } while (stream.next()); pbar->update_percent(100); From 506b0a8a080799c219b3d7cc7890fe36b5efce88 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 11:10:19 +0100 Subject: [PATCH 020/118] [thin_show_dups] inline some hash functions --- base/rolling_hash.cc | 71 +------------------------------------------- base/rolling_hash.h | 60 +++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 75 deletions(-) diff --git a/base/rolling_hash.cc b/base/rolling_hash.cc index 9c6e1bf..8de7ac3 100644 --- a/base/rolling_hash.cc +++ b/base/rolling_hash.cc @@ -2,15 +2,11 @@ using namespace base; using namespace boost; +using namespace hash_detail; using namespace std; //---------------------------------------------------------------- -namespace { - uint32_t MULTIPLIER = 4294967291UL; - uint32_t SEED = 123; -} - rolling_hash::rolling_hash(unsigned window_size) : a_(MULTIPLIER), a_to_k_minus_1_(a_), @@ -35,28 +31,6 @@ rolling_hash::reset() } } -uint32_t -rolling_hash::step(uint8_t byte) -{ - update_hash(byte); - return hash_; -} - -uint32_t -rolling_hash::get_hash() const -{ - return hash_; -} - -void -rolling_hash::update_hash(uint8_t byte) -{ - hash_ -= a_to_k_minus_1_ * (chars_.front() + SEED); - chars_.pop_front(); - chars_.push_back(byte); - hash_ = (hash_ * a_) + byte + SEED; -} - //-------------------------------- content_based_hash::content_based_hash(unsigned window_size) @@ -79,47 +53,4 @@ content_based_hash::reset() rhash_.reset(); } -optional -content_based_hash::step(uint8_t byte) -{ - optional r; - - rhash_.step(byte); - len_++; - - if (len_ < min_len_) - return r; - - if (hit_break(backup_div_)) - backup_break_ = len_; - - if (hit_break(div_)) { - // found a break - r = len_; - len_ = 0; - backup_break_.reset(); - - } else if (len_ >= max_len_) { - // too big, is there a backup? - if (backup_break_) { - len_ -= *backup_break_; - r = backup_break_; - backup_break_.reset(); - - } else { - r = len_; - len_ = 0; - } - } - - return r; -} - -bool -content_based_hash::hit_break(uint32_t mask) const -{ - uint32_t h = rhash_.get_hash() >> 8; - return !(h & mask); -} - //---------------------------------------------------------------- diff --git a/base/rolling_hash.h b/base/rolling_hash.h index d44012a..c5fa44c 100644 --- a/base/rolling_hash.h +++ b/base/rolling_hash.h @@ -8,6 +8,11 @@ //---------------------------------------------------------------- namespace base { + namespace hash_detail { + uint32_t const MULTIPLIER = 4294967291UL; + uint32_t const SEED = 123; + } + class rolling_hash { public: rolling_hash(unsigned window_size); @@ -15,12 +20,22 @@ namespace base { void reset(); // Returns the current hash - uint32_t step(uint8_t byte); + uint32_t step(uint8_t byte) { + update_hash(byte); + return hash_; + } - uint32_t get_hash() const; + uint32_t get_hash() const { + return hash_; + } private: - void update_hash(uint8_t byte); + void update_hash(uint8_t byte) { + hash_ -= a_to_k_minus_1_ * (chars_.front() + hash_detail::SEED); + chars_.pop_front(); + chars_.push_back(byte); + hash_ = (hash_ * a_) + byte + hash_detail::SEED; + } uint32_t a_; uint32_t a_to_k_minus_1_; @@ -38,10 +53,45 @@ namespace base { void reset(); // Returns a break point relative to the last reset/break. - boost::optional step(uint8_t byte); + boost::optional step(uint8_t byte) { + boost::optional r; + + rhash_.step(byte); + len_++; + + if (len_ < min_len_) + return r; + + if (hit_break(backup_div_)) + backup_break_ = len_; + + if (hit_break(div_)) { + // found a break + r = len_; + len_ = 0; + backup_break_.reset(); + + } else if (len_ >= max_len_) { + // too big, is there a backup? + if (backup_break_) { + len_ -= *backup_break_; + r = backup_break_; + backup_break_.reset(); + + } else { + r = len_; + len_ = 0; + } + } + + return r; + } private: - bool hit_break(uint32_t div) const; + bool hit_break(uint32_t mask) const { + uint32_t h = rhash_.get_hash() >> 8; + return !(h & mask); + } rolling_hash rhash_; From 3b9681232873029fdd983874126ba093fb611093 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 11:28:33 +0100 Subject: [PATCH 021/118] [thin_show_dups] switch to boost::circular_buffer in the rolling_hash --- base/rolling_hash.cc | 7 ++++--- base/rolling_hash.h | 12 +++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/base/rolling_hash.cc b/base/rolling_hash.cc index 8de7ac3..c780a27 100644 --- a/base/rolling_hash.cc +++ b/base/rolling_hash.cc @@ -10,7 +10,8 @@ using namespace std; rolling_hash::rolling_hash(unsigned window_size) : a_(MULTIPLIER), a_to_k_minus_1_(a_), - window_size_(window_size) { + window_size_(window_size), + buffer_(window_size) { for (unsigned i = 1; i < window_size_ - 1; i++) a_to_k_minus_1_ *= a_; @@ -22,12 +23,12 @@ void rolling_hash::reset() { // prime with zeroes - chars_.clear(); + buffer_.clear(); hash_ = 0; for (unsigned i = 0; i < window_size_; i++) { hash_ = (hash_ * a_) + SEED; - chars_.push_back(0); + buffer_.push_back(0); } } diff --git a/base/rolling_hash.h b/base/rolling_hash.h index c5fa44c..dff3145 100644 --- a/base/rolling_hash.h +++ b/base/rolling_hash.h @@ -1,7 +1,7 @@ #ifndef BASE_ROLLING_HASH_H #define BASE_ROLLING_HASH_H -#include +#include #include #include @@ -31,20 +31,18 @@ namespace base { private: void update_hash(uint8_t byte) { - hash_ -= a_to_k_minus_1_ * (chars_.front() + hash_detail::SEED); - chars_.pop_front(); - chars_.push_back(byte); + hash_ -= a_to_k_minus_1_ * (buffer_.front() + hash_detail::SEED); + buffer_.push_back(byte); hash_ = (hash_ * a_) + byte + hash_detail::SEED; } uint32_t a_; uint32_t a_to_k_minus_1_; - // FIXME: use a ring buffer - std::list chars_; - uint32_t hash_; uint32_t window_size_; + + boost::circular_buffer buffer_; }; class content_based_hash { From 216e5acb6c32a18da1efa1b7e5f87ed0710426b9 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 13:48:02 +0100 Subject: [PATCH 022/118] [thin_show_dups] remove variable number of mems per chunks. Too slow and not used. --- base/rolling_hash.cc | 2 +- thin-provisioning/cache_stream.cc | 4 ++-- thin-provisioning/chunk_stream.cc | 14 -------------- thin-provisioning/chunk_stream.h | 9 ++++++--- thin-provisioning/thin_show_duplicates.cc | 11 ++++------- thin-provisioning/variable_chunk_stream.cc | 18 +++++++++--------- 6 files changed, 22 insertions(+), 36 deletions(-) diff --git a/base/rolling_hash.cc b/base/rolling_hash.cc index c780a27..d2d273a 100644 --- a/base/rolling_hash.cc +++ b/base/rolling_hash.cc @@ -40,7 +40,7 @@ content_based_hash::content_based_hash(unsigned window_size) // FIXME: hard coded values backup_div_((window_size / 4) - 1), div_((window_size / 2) - 1), - min_len_(window_size / 8), + min_len_(window_size / 4), max_len_(window_size), len_(0) { diff --git a/thin-provisioning/cache_stream.cc b/thin-provisioning/cache_stream.cc index 8fbcc72..b21f435 100644 --- a/thin-provisioning/cache_stream.cc +++ b/thin-provisioning/cache_stream.cc @@ -93,8 +93,8 @@ cache_stream::chunk_wrapper::chunk_wrapper(cache_stream &parent) { c_.offset_ = parent.current_index_ * parent.block_size_; c_.len_ = parent.block_size_; - c_.mem_.push_back(mem(static_cast(block_.get_data()), - static_cast(block_.get_data()) + parent.block_size_)); + c_.mem_.begin = static_cast(block_.get_data()); + c_.mem_.end = c_.mem_.begin + parent.block_size_; } //---------------------------------------------------------------- diff --git a/thin-provisioning/chunk_stream.cc b/thin-provisioning/chunk_stream.cc index 4ac99ff..adc41d0 100644 --- a/thin-provisioning/chunk_stream.cc +++ b/thin-provisioning/chunk_stream.cc @@ -5,19 +5,5 @@ using namespace thin_provisioning; //---------------------------------------------------------------- -uint8_t -chunk::operator[](uint64_t n) const -{ - std::deque::const_iterator it; - for (it = mem_.begin(); it != mem_.end(); it++) { - uint64_t mem_len = it->end - it->begin; - if (n > mem_len) - n -= mem_len; - else - return it->begin[n]; - } - - throw runtime_error("chunk out of bounds"); -} //---------------------------------------------------------------- diff --git a/thin-provisioning/chunk_stream.h b/thin-provisioning/chunk_stream.h index 4e1ea96..0886c9f 100644 --- a/thin-provisioning/chunk_stream.h +++ b/thin-provisioning/chunk_stream.h @@ -28,6 +28,11 @@ namespace thin_provisioning { struct mem { + mem() + : begin(0), + end(0) { + } + mem(uint8_t *b, uint8_t *e) : begin(b), end(e) { @@ -38,9 +43,7 @@ namespace thin_provisioning { struct chunk { uint64_t offset_, len_; - std::deque mem_; - - uint8_t operator[](uint64_t n) const; + mem mem_; }; class chunk_stream { diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 9fc96c2..5a75e26 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -142,8 +142,7 @@ namespace { else { digestor_.reset(); - for (deque::const_iterator it = c.mem_.begin(); it != c.mem_.end(); it++) - digestor_.process_bytes(it->begin, it->end - it->begin); + digestor_.process_bytes(c.mem_.begin, c.mem_.end - c.mem_.begin); unsigned int digest[5]; digestor_.get_digest(digest); @@ -167,11 +166,9 @@ namespace { private: bool all_zeroes(chunk const &c) const { - for (deque::const_iterator it = c.mem_.begin(); it != c.mem_.end(); it++) { - for (uint8_t *ptr = it->begin; ptr != it->end; ptr++) { - if (*ptr != 0) - return false; - } + for (uint8_t *ptr = c.mem_.begin; ptr != c.mem_.end; ptr++) { + if (*ptr != 0) + return false; } return true; diff --git a/thin-provisioning/variable_chunk_stream.cc b/thin-provisioning/variable_chunk_stream.cc index 41c6c96..d2f1529 100644 --- a/thin-provisioning/variable_chunk_stream.cc +++ b/thin-provisioning/variable_chunk_stream.cc @@ -59,8 +59,8 @@ variable_chunk_stream::get() little_chunk_.len_ = little_e_ - little_b_; little_chunk_.offset_ = big_chunk_->offset_ + little_chunk_.len_; - little_chunk_.mem_.clear(); - little_chunk_.mem_.push_back(mem(little_b_, little_e_)); + little_chunk_.mem_.begin = little_b_; + little_chunk_.mem_.end = little_e_; return little_chunk_; } @@ -80,7 +80,7 @@ variable_chunk_stream::next_big_chunk() return false; big_chunk_ = &stream_.get(); - little_b_ = little_e_ = last_hashed_ = big_chunk_->mem_.front().begin; + little_b_ = little_e_ = last_hashed_ = big_chunk_->mem_.begin; h_.reset(); return true; @@ -93,19 +93,19 @@ variable_chunk_stream::advance_one() assert(big_chunk_); - big_e = big_chunk_->mem_.front().end; + big_e = big_chunk_->mem_.end; little_b_ = little_e_; little_e_ = last_hashed_; if (little_b_ == big_e) { if (next_big_chunk()) - big_e = big_chunk_->mem_.front().end; + big_e = big_chunk_->mem_.end; else return false; } - assert(little_e_ >= big_chunk_->mem_.front().begin); - assert(little_b_ >= big_chunk_->mem_.front().begin); + assert(little_e_ >= big_chunk_->mem_.begin); + assert(little_b_ >= big_chunk_->mem_.begin); assert(little_e_ <= big_e); assert(little_b_ <= big_e); @@ -126,8 +126,8 @@ variable_chunk_stream::advance_one() if (little_e_ == big_e) last_hashed_ = little_e_; - assert(little_e_ >= big_chunk_->mem_.front().begin); - assert(little_b_ >= big_chunk_->mem_.front().begin); + assert(little_e_ >= big_chunk_->mem_.begin); + assert(little_b_ >= big_chunk_->mem_.begin); assert(little_e_ <= big_e); assert(little_b_ <= big_e); From 41a1b85c2700ac9d88a2158047e1835b5c0214f9 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 13:56:38 +0100 Subject: [PATCH 023/118] [thin_show_dups] take out some old assertions --- thin-provisioning/variable_chunk_stream.cc | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/thin-provisioning/variable_chunk_stream.cc b/thin-provisioning/variable_chunk_stream.cc index d2f1529..99ddc61 100644 --- a/thin-provisioning/variable_chunk_stream.cc +++ b/thin-provisioning/variable_chunk_stream.cc @@ -91,8 +91,6 @@ variable_chunk_stream::advance_one() { uint8_t *big_e; - assert(big_chunk_); - big_e = big_chunk_->mem_.end; little_b_ = little_e_; little_e_ = last_hashed_; @@ -104,12 +102,6 @@ variable_chunk_stream::advance_one() return false; } - assert(little_e_ >= big_chunk_->mem_.begin); - assert(little_b_ >= big_chunk_->mem_.begin); - assert(little_e_ <= big_e); - assert(little_b_ <= big_e); - - while (little_e_ != big_e) { optional maybe_break = h_.step(*little_e_); little_e_++; @@ -126,11 +118,6 @@ variable_chunk_stream::advance_one() if (little_e_ == big_e) last_hashed_ = little_e_; - assert(little_e_ >= big_chunk_->mem_.begin); - assert(little_b_ >= big_chunk_->mem_.begin); - assert(little_e_ <= big_e); - assert(little_b_ <= big_e); - return true; } From 251762e6d94e903f2a82f3415b997cca249d57d2 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 15:16:49 +0100 Subject: [PATCH 024/118] [thin_show_dups] tidy up reporting --- thin-provisioning/cache_stream.cc | 10 +--- thin-provisioning/cache_stream.h | 3 +- thin-provisioning/chunk_stream.h | 2 +- thin-provisioning/pool_stream.cc | 31 +++++------ thin-provisioning/pool_stream.h | 5 +- thin-provisioning/thin_show_duplicates.cc | 64 +++++++++++----------- thin-provisioning/variable_chunk_stream.cc | 12 ++-- thin-provisioning/variable_chunk_stream.h | 3 +- 8 files changed, 58 insertions(+), 72 deletions(-) diff --git a/thin-provisioning/cache_stream.cc b/thin-provisioning/cache_stream.cc index b21f435..379c191 100644 --- a/thin-provisioning/cache_stream.cc +++ b/thin-provisioning/cache_stream.cc @@ -37,9 +37,9 @@ cache_stream::cache_stream(string const &path, } block_address -cache_stream::nr_chunks() const +cache_stream::size() const { - return nr_blocks_; + return nr_blocks_ * block_size_; } void @@ -68,12 +68,6 @@ cache_stream::eof() const return current_index_ >= nr_blocks_; } -block_address -cache_stream::index() const -{ - return current_index_; -} - chunk const & cache_stream::get() { diff --git a/thin-provisioning/cache_stream.h b/thin-provisioning/cache_stream.h index 65c81b1..b7af995 100644 --- a/thin-provisioning/cache_stream.h +++ b/thin-provisioning/cache_stream.h @@ -14,10 +14,9 @@ namespace thin_provisioning { block_address block_size, size_t cache_mem); - block_address nr_chunks() const; + block_address size() const; virtual void rewind(); - virtual block_address index() const; virtual bool next(block_address count = 1ull); virtual bool eof() const; diff --git a/thin-provisioning/chunk_stream.h b/thin-provisioning/chunk_stream.h index 0886c9f..1831f27 100644 --- a/thin-provisioning/chunk_stream.h +++ b/thin-provisioning/chunk_stream.h @@ -51,7 +51,7 @@ namespace thin_provisioning { virtual ~chunk_stream() {} virtual void rewind() = 0; - virtual bcache::block_address index() const = 0; + virtual bcache::block_address size() const = 0; virtual bool next(bcache::block_address count = 1ull) = 0; virtual bool eof() const = 0; diff --git a/thin-provisioning/pool_stream.cc b/thin-provisioning/pool_stream.cc index 21964f9..41a0ab0 100644 --- a/thin-provisioning/pool_stream.cc +++ b/thin-provisioning/pool_stream.cc @@ -41,22 +41,25 @@ pool_stream::pool_stream(cache_stream &stream, transaction_manager::ptr tm, superblock_detail::superblock const &sb, block_address nr_blocks) : stream_(stream), - block_to_thin_(stream.nr_chunks(), UNMAPPED), - nr_mapped_(0) + block_to_thin_(nr_blocks, UNMAPPED), + nr_mapped_(0), + index_(0), + block_size_(sb.data_block_size_ * 512) { init_rmap(tm, sb, nr_blocks); } block_address -pool_stream::nr_chunks() const +pool_stream::size() const { - return nr_mapped_; + return nr_mapped_ * block_size_; } void pool_stream::rewind() { stream_.rewind(); + index_ = 0; } bool @@ -75,12 +78,6 @@ pool_stream::eof() const return stream_.eof(); } -block_address -pool_stream::index() const -{ - return stream_.index(); -} - chunk const & pool_stream::get() { @@ -141,16 +138,14 @@ pool_stream::init_rmap(transaction_manager::ptr tm, bool pool_stream::advance_one() { - block_address new_index = index() + 1; + block_address count = 1; - while (block_to_thin_[new_index] == UNMAPPED && - new_index < nr_chunks()) - new_index++; + while (((index_ + count) < block_to_thin_.size()) && + (block_to_thin_[index_ + count] == UNMAPPED)) + count++; - if (new_index >= nr_chunks()) - return false; - - return stream_.next(new_index - index()); + index_ += count; + return stream_.next(count); } //---------------------------------------------------------------- diff --git a/thin-provisioning/pool_stream.h b/thin-provisioning/pool_stream.h index 71576ed..e419842 100644 --- a/thin-provisioning/pool_stream.h +++ b/thin-provisioning/pool_stream.h @@ -32,11 +32,10 @@ namespace thin_provisioning { transaction_manager::ptr tm, superblock_detail::superblock const &sb, block_address nr_blocks); - block_address nr_chunks() const; + block_address size() const; void rewind(); bool next(block_address count = 1ull); bool eof() const; - block_address index() const; chunk const &get(); void put(chunk const &c); @@ -56,6 +55,8 @@ namespace thin_provisioning { cache_stream &stream_; vector block_to_thin_; block_address nr_mapped_; + block_address index_; + block_address block_size_; }; } diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 5a75e26..5c09af2 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -182,6 +182,34 @@ namespace { duplicate_counter results_; }; + void display_results(chunk_stream const &stream, duplicate_counter const &r) { + block_address meg = 1024 * 1024; + cout << "\n\n" + << stream.size() / meg << "m examined, " + << r.get_non_zeroes() / meg << "m duplicates, " + << r.get_zeroes() / meg << "m zeroes\n"; + } + + void scan(chunk_stream &stream, block_address stream_size) { + duplicate_detector detector; + block_address total_seen(0); + auto_ptr pbar = create_progress_bar("Examining data"); + + do { + // FIXME: use a wrapper class to automate the put() + chunk const &c = stream.get(); + detector.examine(c); + stream.put(c); + + total_seen += c.len_; + pbar->update_percent((total_seen * 100) / stream.size()); + + } while (stream.next()); + + pbar->update_percent(100); + display_results(stream, detector.get_results()); + } + int show_dups_pool(flags const &fs) { block_manager<>::ptr bm = open_bm(*fs.metadata_dev); transaction_manager::ptr tm = open_tm(bm); @@ -195,21 +223,9 @@ namespace { cache_stream stream(fs.data_dev, block_size, fs.cache_mem); pool_stream pstream(stream, tm, sb, nr_blocks); + variable_chunk_stream vstream(pstream, 4096); - duplicate_detector detector; - auto_ptr pbar = create_progress_bar("Examining data"); - - do { - chunk const &c = pstream.get(); - detector.examine(c); - pstream.put(c); - pbar->update_percent((pstream.index() * 100) / pstream.nr_chunks()); - - } while (pstream.next()); - pbar->update_percent(100); - - cout << "\n\ntotal dups: " << detector.get_results().get_total() << endl; - cout << (detector.get_results().get_total() * 100) / pstream.nr_chunks() << "% duplicates\n"; + scan(vstream, nr_blocks * block_size); return 0; } @@ -229,26 +245,8 @@ namespace { cache_stream low_level_stream(fs.data_dev, block_size, fs.cache_mem); variable_chunk_stream stream(low_level_stream, 4096); - duplicate_detector detector; - auto_ptr pbar = create_progress_bar("Examining data"); - do { - // FIXME: use a wrapper class to automate the put() - chunk const &c = stream.get(); - detector.examine(c); - stream.put(c); - - pbar->update_percent((c.offset_ * 100) / dev_size); - - } while (stream.next()); - pbar->update_percent(100); - - duplicate_counter r = detector.get_results(); - block_address meg = 1024 * 1024; - cout << "\n\n" - << (nr_blocks * block_size) / meg << "m examined, " - << r.get_non_zeroes() / meg << "m duplicates, " - << r.get_zeroes() / meg << "m zeroes\n"; + scan(stream, dev_size); return 0; } diff --git a/thin-provisioning/variable_chunk_stream.cc b/thin-provisioning/variable_chunk_stream.cc index 99ddc61..f572db7 100644 --- a/thin-provisioning/variable_chunk_stream.cc +++ b/thin-provisioning/variable_chunk_stream.cc @@ -19,6 +19,12 @@ variable_chunk_stream::~variable_chunk_stream() put_big_chunk(); } +bcache::block_address +variable_chunk_stream::size() const +{ + return stream_.size(); +} + void variable_chunk_stream::rewind() { @@ -45,12 +51,6 @@ variable_chunk_stream::eof() const return stream_.eof(); } -bcache::block_address -variable_chunk_stream::index() const -{ - return index_; -} - chunk const & variable_chunk_stream::get() { diff --git a/thin-provisioning/variable_chunk_stream.h b/thin-provisioning/variable_chunk_stream.h index f9c5ec7..cc62945 100644 --- a/thin-provisioning/variable_chunk_stream.h +++ b/thin-provisioning/variable_chunk_stream.h @@ -13,11 +13,10 @@ namespace thin_provisioning { variable_chunk_stream(chunk_stream &stream, unsigned window_size); ~variable_chunk_stream(); - // FIXME: we don't know in advance how many chunks we will have + virtual bcache::block_address size() const; virtual void rewind(); virtual bool next(bcache::block_address count = 1ull); virtual bool eof() const; - virtual bcache::block_address index() const; virtual chunk const &get(); virtual void put(chunk const &c); From cb56b474000edd45403bf09662a74254027e73be Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 4 Sep 2015 15:27:48 +0100 Subject: [PATCH 025/118] [thin-show-dups] add --content-based-chunks --- thin-provisioning/thin_show_duplicates.cc | 38 +++++++++++++++-------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 5c09af2..fbcdd10 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -90,13 +90,15 @@ namespace { struct flags { flags() - : cache_mem(64 * 1024 * 1024) { + : cache_mem(64 * 1024 * 1024), + content_based_chunks(false) { } string data_dev; optional metadata_dev; optional block_size; unsigned cache_mem; + bool content_based_chunks; }; using namespace mapping_tree_detail; @@ -190,7 +192,7 @@ namespace { << r.get_zeroes() / meg << "m zeroes\n"; } - void scan(chunk_stream &stream, block_address stream_size) { + void scan(chunk_stream &stream) { duplicate_detector detector; block_address total_seen(0); auto_ptr pbar = create_progress_bar("Examining data"); @@ -210,6 +212,11 @@ namespace { display_results(stream, detector.get_results()); } + void scan_with_variable_sized_chunks(chunk_stream &stream) { + variable_chunk_stream vstream(stream, 4096); + scan(vstream); + } + int show_dups_pool(flags const &fs) { block_manager<>::ptr bm = open_bm(*fs.metadata_dev); transaction_manager::ptr tm = open_tm(bm); @@ -217,15 +224,13 @@ namespace { block_address block_size = sb.data_block_size_ * 512; block_address nr_blocks = get_nr_blocks(fs.data_dev, block_size); - cerr << "path = " << fs.data_dev << "\n"; - cerr << "block size = " << block_size << "\n"; - cerr << "nr_blocks = " << nr_blocks << "\n"; - cache_stream stream(fs.data_dev, block_size, fs.cache_mem); pool_stream pstream(stream, tm, sb, nr_blocks); - variable_chunk_stream vstream(pstream, 4096); - scan(vstream, nr_blocks * block_size); + if (fs.content_based_chunks) + scan_with_variable_sized_chunks(pstream); + else + scan(pstream); return 0; } @@ -237,16 +242,17 @@ namespace { block_address block_size = *fs.block_size; block_address nr_blocks = get_nr_blocks(fs.data_dev, *fs.block_size); - block_address dev_size = nr_blocks * *fs.block_size; cerr << "path = " << fs.data_dev << "\n"; cerr << "nr_blocks = " << nr_blocks << "\n"; cerr << "block size = " << block_size << "\n"; - cache_stream low_level_stream(fs.data_dev, block_size, fs.cache_mem); - variable_chunk_stream stream(low_level_stream, 4096); + cache_stream stream(fs.data_dev, block_size, fs.cache_mem); - scan(stream, dev_size); + if (fs.content_based_chunks) + scan_with_variable_sized_chunks(stream); + else + scan(stream); return 0; } @@ -264,6 +270,7 @@ namespace { out << "Usage: " << cmd << " [options] {device|file}\n" << "Options:\n" << " {--block-sectors} \n" + << " {--content-based-chunks}\n" << " {--metadata-dev} \n" << " {-h|--help}\n" << " {-V|--version}" << endl; @@ -278,7 +285,8 @@ int thin_show_dups_main(int argc, char **argv) char const shortopts[] = "qhV"; option const longopts[] = { { "block-sectors", required_argument, NULL, 1}, - { "metadata-dev", required_argument, NULL, 2}, + { "content-based-chunks", no_argument, NULL, 2}, + { "metadata-dev", required_argument, NULL, 3}, { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } @@ -299,6 +307,10 @@ int thin_show_dups_main(int argc, char **argv) break; case 2: + fs.content_based_chunks = true; + break; + + case 3: fs.metadata_dev = optarg; break; From b6e3a12297943616043bfcfb86de13b36941e7f4 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 7 Sep 2015 15:40:35 +0100 Subject: [PATCH 026/118] [thin_show_dups] move scan into the duplicate_detector --- thin-provisioning/thin_show_duplicates.cc | 82 ++++++++++++----------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index fbcdd10..5b6b650 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -130,6 +130,14 @@ namespace { return zero_dups_; } + void display_results(chunk_stream const &stream) const { + block_address meg = 1024 * 1024; + cout << "\n\n" + << stream.size() / meg << "m examined, " + << get_non_zeroes() / meg << "m duplicates, " + << get_zeroes() / meg << "m zeroes\n"; + } + private: block_address non_zero_dups_; block_address zero_dups_; @@ -137,13 +145,36 @@ namespace { class duplicate_detector { public: + void scan(chunk_stream &stream) { + block_address total_seen(0); + auto_ptr pbar = create_progress_bar("Examining data"); + + do { + // FIXME: use a wrapper class to automate the put() + chunk const &c = stream.get(); + examine(c); + stream.put(c); + + total_seen += c.len_; + pbar->update_percent((total_seen * 100) / stream.size()); + + } while (stream.next()); + + pbar->update_percent(100); + results_.display_results(stream); + } + + duplicate_counter const &get_results() const { + return results_; + } + + private: void examine(chunk const &c) { if (all_zeroes(c)) results_.add_zero_duplicate(c.len_); else { digestor_.reset(); - digestor_.process_bytes(c.mem_.begin, c.mem_.end - c.mem_.begin); unsigned int digest[5]; @@ -162,11 +193,6 @@ namespace { } } - duplicate_counter const &get_results() const { - return results_; - } - - private: bool all_zeroes(chunk const &c) const { for (uint8_t *ptr = c.mem_.begin; ptr != c.mem_.end; ptr++) { if (*ptr != 0) @@ -184,37 +210,10 @@ namespace { duplicate_counter results_; }; - void display_results(chunk_stream const &stream, duplicate_counter const &r) { - block_address meg = 1024 * 1024; - cout << "\n\n" - << stream.size() / meg << "m examined, " - << r.get_non_zeroes() / meg << "m duplicates, " - << r.get_zeroes() / meg << "m zeroes\n"; - } - - void scan(chunk_stream &stream) { - duplicate_detector detector; - block_address total_seen(0); - auto_ptr pbar = create_progress_bar("Examining data"); - - do { - // FIXME: use a wrapper class to automate the put() - chunk const &c = stream.get(); - detector.examine(c); - stream.put(c); - - total_seen += c.len_; - pbar->update_percent((total_seen * 100) / stream.size()); - - } while (stream.next()); - - pbar->update_percent(100); - display_results(stream, detector.get_results()); - } - - void scan_with_variable_sized_chunks(chunk_stream &stream) { + void scan_with_variable_sized_chunks(chunk_stream &stream, + duplicate_detector &detector) { variable_chunk_stream vstream(stream, 4096); - scan(vstream); + detector.scan(vstream); } int show_dups_pool(flags const &fs) { @@ -227,10 +226,12 @@ namespace { cache_stream stream(fs.data_dev, block_size, fs.cache_mem); pool_stream pstream(stream, tm, sb, nr_blocks); + duplicate_detector detector; + if (fs.content_based_chunks) - scan_with_variable_sized_chunks(pstream); + scan_with_variable_sized_chunks(pstream, detector); else - scan(pstream); + detector.scan(pstream); return 0; } @@ -248,11 +249,12 @@ namespace { cerr << "block size = " << block_size << "\n"; cache_stream stream(fs.data_dev, block_size, fs.cache_mem); + duplicate_detector dd; if (fs.content_based_chunks) - scan_with_variable_sized_chunks(stream); + scan_with_variable_sized_chunks(stream, dd); else - scan(stream); + dd.scan(stream); return 0; } From c58c15e7883e755d756702742fc2146670a5adf8 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 8 Sep 2015 13:17:52 +0100 Subject: [PATCH 027/118] [thin_show_dups] move scan_with_variable_sized_chunks() into the dup detector --- thin-provisioning/thin_show_duplicates.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 5b6b650..3352760 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -164,6 +164,12 @@ namespace { results_.display_results(stream); } + + void scan_with_variable_sized_chunks(chunk_stream &stream) { + variable_chunk_stream vstream(stream, 4096); + scan(vstream); + } + duplicate_counter const &get_results() const { return results_; } @@ -210,12 +216,6 @@ namespace { duplicate_counter results_; }; - void scan_with_variable_sized_chunks(chunk_stream &stream, - duplicate_detector &detector) { - variable_chunk_stream vstream(stream, 4096); - detector.scan(vstream); - } - int show_dups_pool(flags const &fs) { block_manager<>::ptr bm = open_bm(*fs.metadata_dev); transaction_manager::ptr tm = open_tm(bm); @@ -229,7 +229,7 @@ namespace { duplicate_detector detector; if (fs.content_based_chunks) - scan_with_variable_sized_chunks(pstream, detector); + detector.scan_with_variable_sized_chunks(pstream); else detector.scan(pstream); @@ -252,7 +252,7 @@ namespace { duplicate_detector dd; if (fs.content_based_chunks) - scan_with_variable_sized_chunks(stream, dd); + dd.scan_with_variable_sized_chunks(stream); else dd.scan(stream); From 664841ad03506ba1da2b44d9e23623e09f3f3285 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 8 Sep 2015 17:09:41 +0100 Subject: [PATCH 028/118] [thin_show_dups] Support fractions of a pool block size --- Makefile.in | 1 + thin-provisioning/fixed_chunk_stream.cc | 113 ++++++++++++++++++++++ thin-provisioning/fixed_chunk_stream.h | 39 ++++++++ thin-provisioning/thin_show_duplicates.cc | 42 +++++--- 4 files changed, 180 insertions(+), 15 deletions(-) create mode 100644 thin-provisioning/fixed_chunk_stream.cc create mode 100644 thin-provisioning/fixed_chunk_stream.h diff --git a/Makefile.in b/Makefile.in index d09c873..89d45fd 100644 --- a/Makefile.in +++ b/Makefile.in @@ -75,6 +75,7 @@ SOURCE=\ thin-provisioning/cache_stream.cc \ thin-provisioning/chunk_stream.cc \ thin-provisioning/device_tree.cc \ + thin-provisioning/fixed_chunk_stream.cc \ thin-provisioning/human_readable_format.cc \ thin-provisioning/mapping_tree.cc \ thin-provisioning/metadata.cc \ diff --git a/thin-provisioning/fixed_chunk_stream.cc b/thin-provisioning/fixed_chunk_stream.cc new file mode 100644 index 0000000..ea031dd --- /dev/null +++ b/thin-provisioning/fixed_chunk_stream.cc @@ -0,0 +1,113 @@ +#include "thin-provisioning/fixed_chunk_stream.h" + +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +fixed_chunk_stream::fixed_chunk_stream(chunk_stream &stream, unsigned chunk_size) + : index_(0), + stream_(stream), + chunk_size_(chunk_size), + big_chunk_(0) { + next_big_chunk(); +} + +fixed_chunk_stream::~fixed_chunk_stream() +{ + put_big_chunk(); +} + +bcache::block_address +fixed_chunk_stream::size() const +{ + return stream_.size(); +} + +void +fixed_chunk_stream::rewind() +{ + // FIXME: not complete + index_ = 0; + stream_.rewind(); +} + +bool +fixed_chunk_stream::next(bcache::block_address count) +{ + while (count--) { + index_++; + advance_one(); + } + + return !eof(); +} + +bool +fixed_chunk_stream::eof() const +{ + return stream_.eof(); +} + +chunk const & +fixed_chunk_stream::get() +{ + assert(big_chunk_); + + little_chunk_.len_ = little_e_ - little_b_; + little_chunk_.offset_ = big_chunk_->offset_ + little_chunk_.len_; + + little_chunk_.mem_.begin = little_b_; + little_chunk_.mem_.end = little_e_; + + return little_chunk_; +} + +void +fixed_chunk_stream::put(chunk const &c) +{ + // noop +} + +bool +fixed_chunk_stream::next_big_chunk() +{ + put_big_chunk(); + + if (!stream_.next()) + return false; + + big_chunk_ = &stream_.get(); + little_b_ = little_e_ = last_hashed_ = big_chunk_->mem_.begin; + + return true; +} + +bool +fixed_chunk_stream::advance_one() +{ + uint8_t *big_e; + + big_e = big_chunk_->mem_.end; + little_b_ = little_e_; + + if (little_b_ >= big_e) { + if (next_big_chunk()) + big_e = big_chunk_->mem_.end; + else + return false; + } + + little_e_ += chunk_size_; + return true; +} + +void +fixed_chunk_stream::put_big_chunk() +{ + if (big_chunk_) + stream_.put(*big_chunk_); + + big_chunk_ = 0; +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/fixed_chunk_stream.h b/thin-provisioning/fixed_chunk_stream.h new file mode 100644 index 0000000..f17d15a --- /dev/null +++ b/thin-provisioning/fixed_chunk_stream.h @@ -0,0 +1,39 @@ +#ifndef THIN_PROVISIONING_FIXED_CHUNK_STREAM_H +#define THIN_PROVISIONING_FIXED_CHUNK_STREAM_H + +#include "thin-provisioning/chunk_stream.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + class fixed_chunk_stream : public chunk_stream { + public: + fixed_chunk_stream(chunk_stream &stream, unsigned chunk_size); + ~fixed_chunk_stream(); + + virtual bcache::block_address size() const; + virtual void rewind(); + virtual bool next(bcache::block_address count = 1ull); + virtual bool eof() const; + virtual chunk const &get(); + virtual void put(chunk const &c); + + private: + bool next_big_chunk(); + bool advance_one(); + void put_big_chunk(); + + bcache::block_address index_; + + chunk_stream &stream_; + unsigned chunk_size_; + chunk const *big_chunk_; + + uint8_t *little_b_, *little_e_, *last_hashed_; + chunk little_chunk_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/thin_show_duplicates.cc b/thin-provisioning/thin_show_duplicates.cc index 3352760..89c0c9d 100644 --- a/thin-provisioning/thin_show_duplicates.cc +++ b/thin-provisioning/thin_show_duplicates.cc @@ -30,6 +30,7 @@ #include "persistent-data/space-maps/core.h" #include "persistent-data/space-maps/disk.h" #include "thin-provisioning/cache_stream.h" +#include "thin-provisioning/fixed_chunk_stream.h" #include "thin-provisioning/pool_stream.h" #include "thin-provisioning/commands.h" #include "thin-provisioning/device_tree.h" @@ -54,7 +55,6 @@ using namespace thin_provisioning; namespace { bool factor_of(block_address f, block_address n) { - cerr << n << " % " << f << "\n"; return (n % f) == 0; } @@ -145,6 +145,21 @@ namespace { class duplicate_detector { public: + void scan_with_variable_sized_chunks(chunk_stream &stream) { + variable_chunk_stream vstream(stream, 4096); + scan(vstream); + } + + void scan_with_fixed_sized_chunks(chunk_stream &stream, block_address chunk_size) { + fixed_chunk_stream fstream(stream, chunk_size); + scan(fstream); + } + + duplicate_counter const &get_results() const { + return results_; + } + + private: void scan(chunk_stream &stream) { block_address total_seen(0); auto_ptr pbar = create_progress_bar("Examining data"); @@ -164,17 +179,6 @@ namespace { results_.display_results(stream); } - - void scan_with_variable_sized_chunks(chunk_stream &stream) { - variable_chunk_stream vstream(stream, 4096); - scan(vstream); - } - - duplicate_counter const &get_results() const { - return results_; - } - - private: void examine(chunk const &c) { if (all_zeroes(c)) results_.add_zero_duplicate(c.len_); @@ -230,8 +234,16 @@ namespace { if (fs.content_based_chunks) detector.scan_with_variable_sized_chunks(pstream); - else - detector.scan(pstream); + else { + if (*fs.block_size) { + if (factor_of(*fs.block_size, block_size)) + block_size = *fs.block_size; + else + throw runtime_error("specified block size is not a factor of the pool chunk size\n"); + } + + detector.scan_with_fixed_sized_chunks(pstream, block_size); + } return 0; } @@ -254,7 +266,7 @@ namespace { if (fs.content_based_chunks) dd.scan_with_variable_sized_chunks(stream); else - dd.scan(stream); + dd.scan_with_fixed_sized_chunks(stream, block_size); return 0; } From 639af9c3bf79e1267014e0cbf776adc98a2baeb5 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 4 Feb 2016 08:57:41 +0000 Subject: [PATCH 029/118] [block-cache] convert to use boost::intrusive rather than kernel style lists. Major change, do not release until a lot of testing has been done. Conflicts: block-cache/block_cache.h --- block-cache/block_cache.cc | 220 ++++++++++++------------------------- block-cache/block_cache.h | 139 ++++++++++++++++++----- 2 files changed, 183 insertions(+), 176 deletions(-) diff --git a/block-cache/block_cache.cc b/block-cache/block_cache.cc index 6ecce1f..e56c46c 100644 --- a/block-cache/block_cache.cc +++ b/block-cache/block_cache.cc @@ -1,5 +1,6 @@ #include "block-cache/block_cache.h" +#include #include #include #include @@ -44,34 +45,19 @@ namespace { int block_cache::init_free_list(unsigned count) { - size_t len; - block *blocks; size_t block_size = block_size_ << SECTOR_SHIFT; - void *data; - unsigned i; - - /* Allocate the block structures */ - len = sizeof(block) * count; - blocks = static_cast(malloc(len)); - if (!blocks) - return -ENOMEM; - - blocks_memory_ = blocks; + unsigned char *data = static_cast(alloc_aligned(count * block_size, PAGE_SIZE)); /* Allocate the data for each block. We page align the data. */ - data = alloc_aligned(count * block_size, PAGE_SIZE); - if (!data) { - free(blocks); + if (!data) return -ENOMEM; - } blocks_data_ = data; - for (i = 0; i < count; i++) { - block *b = new (blocks + i) block(); - b->data_ = static_cast(data) + block_size * i; - - list_add(&b->list_, &free_); + for (unsigned i = 0; i < count; i++) { + block &b = (*blocks_memory_)[i]; + b.data_ = data + (block_size * i); + free_.push_front(b); } return 0; @@ -82,28 +68,18 @@ block_cache::exit_free_list() { if (blocks_data_) free(blocks_data_); - - if (blocks_memory_) { - struct block *blocks = static_cast(blocks_memory_); - for (unsigned i = 0; i < nr_cache_blocks_; i++) - (blocks + i)->~block(); - - free(blocks_memory_); - } } block_cache::block * block_cache::__alloc_block() { - block *b; - - if (list_empty(&free_)) + if (free_.empty()) return NULL; - b = list_first_entry(&free_, block, list_); - list_del(&b->list_); + block &b = free_.front(); + b.unlink(); - return b; + return &b; } /*---------------------------------------------------------------- @@ -131,15 +107,18 @@ block_cache::complete_io(block &b, int result) b.clear_flags(BF_IO_PENDING); nr_io_pending_--; - if (b.error_) - list_move_tail(&b.list_, &errored_); - else { + if (b.error_) { + b.unlink(); + errored_.push_back(b); + + } else { if (b.test_flags(BF_DIRTY)) { b.clear_flags(BF_DIRTY | BF_PREVIOUSLY_DIRTY); nr_dirty_--; } - list_move_tail(&b.list_, &clean_); + b.unlink(); + clean_.push_back(b); } } @@ -157,7 +136,8 @@ block_cache::issue_low_level(block &b, enum io_iocb_cmd opcode, const char *desc assert(!b.test_flags(BF_IO_PENDING)); b.set_flags(BF_IO_PENDING); nr_io_pending_++; - list_move_tail(&b.list_, &io_pending_); + b.unlink(); + io_pending_.push_back(b); b.control_block_.aio_lio_opcode = opcode; control_blocks[0] = &b.control_block_; @@ -208,7 +188,7 @@ block_cache::wait_io() for (i = 0; i < static_cast(r); i++) { io_event const &e = events_[i]; - block *b = container_of(e.obj, block, control_block_); + block *b = container_of(e.obj, &block::control_block_); if (e.res == block_size_ << SECTOR_SHIFT) complete_io(*b, 0); @@ -236,19 +216,20 @@ block_cache::wait_io() * We're using lru lists atm, but I think it would be worth * experimenting with a multiqueue approach. */ -list_head * +block_cache::block_list & block_cache::__categorise(block &b) { if (b.error_) - return &errored_; + return errored_; - return b.test_flags(BF_DIRTY) ? &dirty_ : &clean_; + return b.test_flags(BF_DIRTY) ? dirty_ : clean_; } void block_cache::hit(block &b) { - list_move_tail(&b.list_, __categorise(b)); + b.unlink(); + __categorise(b).push_back(b); } /*---------------------------------------------------------------- @@ -257,7 +238,7 @@ block_cache::hit(block &b) void block_cache::wait_all() { - while (!list_empty(&io_pending_)) + while (!io_pending_.empty()) wait_io(); } @@ -271,10 +252,15 @@ block_cache::wait_specific(block &b) unsigned block_cache::writeback(unsigned count) { - block *b, *tmp; unsigned actual = 0, dirty_length = 0; - list_for_each_entry_safe (b, tmp, &dirty_, list_) { + // issue_write unlinks b, which invalidates the iteration, so we + // keep track of the next element before removing. + auto it = dirty_.begin(); + auto next = it; + while (it != dirty_.end()) { + next = it; + ++next; dirty_length++; if (actual == count) @@ -282,69 +268,18 @@ block_cache::writeback(unsigned count) // The block may be on the dirty list from a prior // acquisition. - if (b->ref_count_) + if (it->ref_count_) continue; - issue_write(*b); + issue_write(*it); actual++; + + it = next; } return actual; } -/*---------------------------------------------------------------- - * Hash table - *---------------------------------------------------------------*/ - -/* - * |nr_buckets| must be a power of two. - */ -void -block_cache::hash_init(unsigned nr_buckets) -{ - unsigned i; - - nr_buckets_ = nr_buckets; - mask_ = nr_buckets - 1; - - for (i = 0; i < nr_buckets; i++) - INIT_LIST_HEAD(&buckets_[i]); -} - -unsigned -block_cache::hash(uint64_t index) -{ - const unsigned BIG_PRIME = 4294967291UL; - return (((unsigned) index) * BIG_PRIME) & mask_; -} - -block_cache::block * -block_cache::hash_lookup(block_address index) -{ - block *b; - unsigned bucket = hash(index); - - list_for_each_entry (b, &buckets_[bucket], hash_list_) { - if (b->index_ == index) - return b; - } - - return NULL; -} - -void -block_cache::hash_insert(block &b) -{ - unsigned bucket = hash(b.index_); - list_move_tail(&b.hash_list_, &buckets_[bucket]); -} - -void -block_cache::hash_remove(block &b) -{ - list_del_init(&b.hash_list_); -} - /*---------------------------------------------------------------- * High level allocation *--------------------------------------------------------------*/ @@ -362,18 +297,17 @@ block_cache::setup_control_block(block &b) cb->u.c.nbytes = block_size_bytes; } +// FIXME: return a reference block_cache::block * block_cache::find_unused_clean_block() { - struct block *b, *tmp; - - list_for_each_entry_safe (b, tmp, &clean_, list_) { - if (b->ref_count_) + for (block &b : clean_) { + if (b.ref_count_) continue; - hash_remove(*b); - list_del(&b->list_); - return b; + block_set_.remove_node(b); + b.unlink(); + return &b; } return NULL; @@ -386,8 +320,8 @@ block_cache::new_block(block_address index) b = __alloc_block(); if (!b) { - if (list_empty(&clean_)) { - if (list_empty(&io_pending_)) + if (clean_.empty()) { + if (io_pending_.empty()) writeback(16); wait_io(); } @@ -396,8 +330,6 @@ block_cache::new_block(block_address index) } if (b) { - INIT_LIST_HEAD(&b->list_); - INIT_LIST_HEAD(&b->hash_list_); b->bc_ = this; b->ref_count_ = 0; @@ -408,7 +340,7 @@ block_cache::new_block(block_address index) b->index_ = index; setup_control_block(*b); - hash_insert(*b); + block_set_.insert(*b); } return b; @@ -455,9 +387,6 @@ block_cache::block_cache(int fd, sector_t block_size, uint64_t on_disk_blocks, s { int r; unsigned nr_cache_blocks = calc_nr_cache_blocks(mem, block_size); - unsigned nr_buckets = calc_nr_buckets(nr_cache_blocks); - - buckets_.resize(nr_buckets); fd_ = fd; block_size_ = block_size; @@ -473,12 +402,7 @@ block_cache::block_cache(int fd, sector_t block_size, uint64_t on_disk_blocks, s throw std::runtime_error("io_setup failed"); } - hash_init(nr_buckets); - INIT_LIST_HEAD(&free_); - INIT_LIST_HEAD(&errored_); - INIT_LIST_HEAD(&dirty_); - INIT_LIST_HEAD(&clean_); - INIT_LIST_HEAD(&io_pending_); + blocks_memory_.reset(new std::vector(nr_cache_blocks)); r = init_free_list(nr_cache_blocks); if (r) @@ -552,30 +476,31 @@ block_cache::block * block_cache::lookup_or_read_block(block_address index, unsigned flags, validator::ptr v) { - block *b = hash_lookup(index); + auto it = block_set_.find(index, cmp_index()); - if (b) { - if (b->test_flags(BF_IO_PENDING)) { + if (it != block_set_.end()) { + if (it->test_flags(BF_IO_PENDING)) { inc_miss_counter(flags); - wait_specific(*b); + wait_specific(*it); } else inc_hit_counter(flags); if (flags & GF_ZERO) - zero_block(*b); + zero_block(*it); else { - if (b->v_.get() != v.get()) { - if (b->test_flags(BF_DIRTY)) - b->v_->prepare(b->data_, b->index_); - v->check(b->data_, b->index_); + if (it->v_.get() != v.get()) { + if (it->test_flags(BF_DIRTY)) + it->v_->prepare(it->data_, it->index_); + v->check(it->data_, it->index_); } } - b->v_ = v; + it->v_ = v; + return &(*it); } else { inc_miss_counter(flags); - b = new_block(index); + block *b = new_block(index); if (b) { if (flags & GF_ZERO) zero_block(*b); @@ -587,9 +512,9 @@ block_cache::lookup_or_read_block(block_address index, unsigned flags, b->v_ = v; } - } - return (!b || b->error_) ? NULL : b; + return (!b || b->error_) ? NULL : b; + } } block_cache::block & @@ -644,7 +569,8 @@ block_cache::release(block_cache::block &b) if (b.test_flags(BF_DIRTY)) { if (!b.test_flags(BF_PREVIOUSLY_DIRTY)) { - list_move_tail(&b.list_, &dirty_); + b.unlink(); + dirty_.push_back(b); nr_dirty_++; b.set_flags(BF_PREVIOUSLY_DIRTY); } @@ -661,19 +587,18 @@ block_cache::release(block_cache::block &b) int block_cache::flush() { - block *b, *tmp; - - list_for_each_entry_safe (b, tmp, &dirty_, list_) { - if (b->ref_count_ || b->test_flags(BF_IO_PENDING)) + while (!dirty_.empty()) { + block &b = dirty_.front(); + if (b.ref_count_ || b.test_flags(BF_IO_PENDING)) // The superblock may well be still locked. continue; - issue_write(*b); + issue_write(b); } wait_all(); - return list_empty(&errored_) ? 0 : -EIO; + return errored_.empty() ? 0 : -EIO; } void @@ -681,11 +606,12 @@ block_cache::prefetch(block_address index) { check_index(index); - block *b = hash_lookup(index); - if (!b) { + auto it = block_set_.find(index, cmp_index()); + + if (it == block_set_.end()) { prefetches_++; - b = new_block(index); + block *b = new_block(index); if (b) issue_read(*b); } diff --git a/block-cache/block_cache.h b/block-cache/block_cache.h index 4bc6667..61f0646 100644 --- a/block-cache/block_cache.h +++ b/block-cache/block_cache.h @@ -1,18 +1,36 @@ #ifndef BLOCK_CACHE_H #define BLOCK_CACHE_H -#include "block-cache/list.h" - -#include +#include +#include #include - -#include +#include +#include +#include #include #include +#include #include #include #include +namespace bi = boost::intrusive; + +//---------------------------------------------------------------- + +// FIXME: move to own file in base +template +size_t offsetof__(const M P::*member) +{ + return (size_t) &( reinterpret_cast(0)->*member); +} + +template +P *container_of(M *ptr, M const P::*member) +{ + return (P *)((char *)(ptr) - offsetof__(member)); +} + //---------------------------------------------------------------- namespace bcache { @@ -50,7 +68,14 @@ namespace bcache { public: block() : v_() { - INIT_LIST_HEAD(&list_); + } + + bool operator <(block const &rhs) const { + return index_ > rhs.index_; + } + + bool operator ==(block const &rhs) const { + return index_ == rhs.index_; } // Do not give this class a destructor, it wont get @@ -92,16 +117,21 @@ namespace bcache { bc_->release(*this); } + void unlink() { + list_hook_.unlink(); + } + private: friend class block_cache; + friend class cmp_index; block_cache *bc_; uint64_t index_; void *data_; - list_head list_; - list_head hash_list_; + bi::list_member_hook> list_hook_; + bi::set_member_hook<> set_hook_; unsigned ref_count_; @@ -112,6 +142,57 @@ namespace bcache { validator::ptr v_; }; +<<<<<<< HEAD +======= + struct cmp_index { + bool operator()(block_address index, block const &b) const { + return index > b.index_; + } + + bool operator()(block const &b, block_address index) const { + return b.index_ > index; + } + }; + + class auto_block { + public: + auto_block() + : b_(0) { + } + + auto_block(block &b) + : b_(&b) { + } + + ~auto_block() { + put(); + } + + auto_block &operator =(block &b) { + put(); + b_ = &b; + return *this; + } + + void *get_data() const { + if (b_) + return b_->get_data(); + + throw std::runtime_error("auto_block not set"); + } + + private: + void put() { + if (b_) { + b_->put(); + b_ = 0; + } + } + + block *b_; + }; + +>>>>>>> 7fadc34... [block-cache] convert to use boost::intrusive rather than kernel style lists. //-------------------------------- block_cache(int fd, sector_t block_size, @@ -137,24 +218,24 @@ namespace bcache { void prefetch(block_address index); private: + typedef bi::member_hook>, + &block::list_hook_> list_hook_option; + typedef bi::list> block_list; + int init_free_list(unsigned count); - void exit_free_list(); block *__alloc_block(); void complete_io(block &b, int result); void issue_low_level(block &b, enum io_iocb_cmd opcode, const char *desc); void issue_read(block &b); void issue_write(block &b); void wait_io(); - list_head *__categorise(block &b); + block_list &__categorise(block &b); void hit(block &b); void wait_all(); void wait_specific(block &b); unsigned writeback(unsigned count); - void hash_init(unsigned nr_buckets); - unsigned hash(uint64_t index); - block *hash_lookup(block_address index); - void hash_insert(block &b); - void hash_remove(block &b); void setup_control_block(block &b); block *find_unused_clean_block(); block *new_block(block_address index); @@ -163,6 +244,7 @@ namespace bcache { unsigned calc_nr_buckets(unsigned nr_blocks); void zero_block(block &b); block *lookup_or_read_block(block_address index, unsigned flags, validator::ptr v); + void exit_free_list(); void preemptive_writeback(); void release(block_cache::block &block); @@ -178,9 +260,8 @@ namespace bcache { uint64_t nr_data_blocks_; uint64_t nr_cache_blocks_; - // We can't use auto_ptr or unique_ptr because the memory is allocated with malloc - void *blocks_memory_; - void *blocks_data_; + std::unique_ptr> blocks_memory_; + unsigned char *blocks_data_; io_context_t aio_context_; std::vector events_; @@ -189,23 +270,23 @@ namespace bcache { * Blocks on the free list are not initialised, apart from the * b.data field. */ - list_head free_; - list_head errored_; - list_head dirty_; - list_head clean_; + block_list free_; + block_list errored_; + block_list dirty_; + block_list clean_; unsigned nr_locked_; unsigned nr_dirty_; unsigned nr_io_pending_; - struct list_head io_pending_; + block_list io_pending_; - /* - * Hash table fields. - */ - unsigned nr_buckets_; - unsigned mask_; - std::vector buckets_; + typedef bi::member_hook, + &block::set_hook_> block_option; + typedef bi::set> block_set; + block_set block_set_; // Stats unsigned read_hits_; From 767c39cf710652a795bdeda6a8eb9c3ef510c5a0 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 4 Feb 2016 09:02:42 +0000 Subject: [PATCH 030/118] [build] switch to c++11 Conflicts: Makefile.in chunker/cache_stream.cc chunker/cache_stream.h thin-provisioning/thin_archive.cc thin-provisioning/thin_show_duplicates.cc unit-tests/Makefile.in --- Makefile.in | 2 +- base/progress_monitor.cc | 8 ++++---- base/progress_monitor.h | 4 ++-- base/xml_utils.cc | 4 ++-- base/xml_utils.h | 2 +- block-cache/block_cache.cc | 2 +- block-cache/block_cache.h | 11 ++++++----- caching/cache_restore.cc | 4 ++-- thin-provisioning/restore_emitter.cc | 2 +- thin-provisioning/thin_pool.cc | 2 +- unit-tests/btree_t.cc | 12 ++++++------ unit-tests/run_set_t.cc | 2 +- unit-tests/space_map_t.cc | 4 ++-- unit-tests/test_utils.h | 2 +- 14 files changed, 31 insertions(+), 30 deletions(-) diff --git a/Makefile.in b/Makefile.in index 30b0c1a..da0e5c7 100644 --- a/Makefile.in +++ b/Makefile.in @@ -105,7 +105,7 @@ TOP_DIR:=@top_srcdir@ TOP_BUILDDIR:=@top_builddir@ CFLAGS+=-g -Wall -O3 CFLAGS+=@LFS_FLAGS@ -CXXFLAGS+=-g -Wall -fno-strict-aliasing -std=gnu++98 +CXXFLAGS+=-g -Wall -fno-strict-aliasing -std=c++11 CXXFLAGS+=@CXXOPTIMISE_FLAG@ CXXFLAGS+=@CXXDEBUG_FLAG@ CXXFLAGS+=@CXX_STRERROR_FLAG@ diff --git a/base/progress_monitor.cc b/base/progress_monitor.cc index 1d88302..d9f7e48 100644 --- a/base/progress_monitor.cc +++ b/base/progress_monitor.cc @@ -63,16 +63,16 @@ namespace { //---------------------------------------------------------------- -std::auto_ptr +std::unique_ptr base::create_progress_bar(std::string const &title) { - return auto_ptr(new progress_bar(title)); + return unique_ptr(new progress_bar(title)); } -std::auto_ptr +std::unique_ptr base::create_quiet_progress_monitor() { - return auto_ptr(new quiet_progress()); + return unique_ptr(new quiet_progress()); } //---------------------------------------------------------------- diff --git a/base/progress_monitor.h b/base/progress_monitor.h index 5472343..b3152d1 100644 --- a/base/progress_monitor.h +++ b/base/progress_monitor.h @@ -15,8 +15,8 @@ namespace base { virtual void update_percent(unsigned) = 0; }; - std::auto_ptr create_progress_bar(std::string const &title); - std::auto_ptr create_quiet_progress_monitor(); + std::unique_ptr create_progress_bar(std::string const &title); + std::unique_ptr create_quiet_progress_monitor(); } //---------------------------------------------------------------- diff --git a/base/xml_utils.cc b/base/xml_utils.cc index fb34153..6fc27f9 100644 --- a/base/xml_utils.cc +++ b/base/xml_utils.cc @@ -14,7 +14,7 @@ xml_parser::parse(std::string const &backup_file, bool quiet) persistent_data::check_file_exists(backup_file); ifstream in(backup_file.c_str(), ifstream::in); - std::auto_ptr monitor = create_monitor(quiet); + std::unique_ptr monitor = create_monitor(quiet); size_t total = 0; size_t input_length = get_file_length(backup_file); @@ -53,7 +53,7 @@ xml_parser::get_file_length(string const &file) const return info.st_size; } -auto_ptr +unique_ptr xml_parser::create_monitor(bool quiet) { if (!quiet && isatty(fileno(stdout))) diff --git a/base/xml_utils.h b/base/xml_utils.h index f867f56..fbfdd2c 100644 --- a/base/xml_utils.h +++ b/base/xml_utils.h @@ -37,7 +37,7 @@ namespace xml_utils { private: size_t get_file_length(string const &file) const; - auto_ptr create_monitor(bool quiet); + unique_ptr create_monitor(bool quiet); XML_Parser parser_; }; diff --git a/block-cache/block_cache.cc b/block-cache/block_cache.cc index e56c46c..1b489ea 100644 --- a/block-cache/block_cache.cc +++ b/block-cache/block_cache.cc @@ -305,7 +305,7 @@ block_cache::find_unused_clean_block() if (b.ref_count_) continue; - block_set_.remove_node(b); + b.unlink_set(); b.unlink(); return &b; } diff --git a/block-cache/block_cache.h b/block-cache/block_cache.h index 61f0646..8c79a77 100644 --- a/block-cache/block_cache.h +++ b/block-cache/block_cache.h @@ -117,6 +117,10 @@ namespace bcache { bc_->release(*this); } + void unlink_set() { + set_hook_.unlink(); + } + void unlink() { list_hook_.unlink(); } @@ -131,7 +135,7 @@ namespace bcache { void *data_; bi::list_member_hook> list_hook_; - bi::set_member_hook<> set_hook_; + bi::set_member_hook> set_hook_; unsigned ref_count_; @@ -142,8 +146,6 @@ namespace bcache { validator::ptr v_; }; -<<<<<<< HEAD -======= struct cmp_index { bool operator()(block_address index, block const &b) const { return index > b.index_; @@ -192,7 +194,6 @@ namespace bcache { block *b_; }; ->>>>>>> 7fadc34... [block-cache] convert to use boost::intrusive rather than kernel style lists. //-------------------------------- block_cache(int fd, sector_t block_size, @@ -282,7 +283,7 @@ namespace bcache { block_list io_pending_; typedef bi::member_hook, + bi::set_member_hook>, &block::set_hook_> block_option; typedef bi::set> block_set; diff --git a/caching/cache_restore.cc b/caching/cache_restore.cc index a357eed..629aab6 100644 --- a/caching/cache_restore.cc +++ b/caching/cache_restore.cc @@ -32,7 +32,7 @@ namespace { return info.st_size; } - auto_ptr create_monitor(bool quiet) { + unique_ptr create_monitor(bool quiet) { if (!quiet && isatty(fileno(stdout))) return create_progress_bar("Restoring"); else @@ -70,7 +70,7 @@ namespace { check_file_exists(*fs.input); ifstream in(fs.input->c_str(), ifstream::in); - auto_ptr monitor = create_monitor(fs.quiet); + unique_ptr monitor = create_monitor(fs.quiet); parse_xml(in, restorer, get_file_length(*fs.input), *monitor); } catch (std::exception &e) { diff --git a/thin-provisioning/restore_emitter.cc b/thin-provisioning/restore_emitter.cc index fd83f56..5330b62 100644 --- a/thin-provisioning/restore_emitter.cc +++ b/thin-provisioning/restore_emitter.cc @@ -142,7 +142,7 @@ namespace { bool device_exists(thin_dev_t dev) const { uint64_t key[1] = {dev}; device_tree::maybe_value v = md_->details_->lookup(key); - return v; + return !!v; } metadata::ptr md_; diff --git a/thin-provisioning/thin_pool.cc b/thin-provisioning/thin_pool.cc index 1596c90..1abaa57 100644 --- a/thin-provisioning/thin_pool.cc +++ b/thin-provisioning/thin_pool.cc @@ -232,7 +232,7 @@ bool thin_pool::device_exists(thin_dev_t dev) const { uint64_t key[1] = {dev}; - return md_->details_->lookup(key); + return !!md_->details_->lookup(key); } //---------------------------------------------------------------- diff --git a/unit-tests/btree_t.cc b/unit-tests/btree_t.cc index 13a525e..46d5cd3 100644 --- a/unit-tests/btree_t.cc +++ b/unit-tests/btree_t.cc @@ -129,7 +129,7 @@ TEST_F(BtreeTests, insert_works) tree->insert(key, value); btree<1, uint64_traits>::maybe_value l = tree->lookup(key); - ASSERT_TRUE(l); + ASSERT_TRUE(!!l); ASSERT_THAT(*l, Eq(i)); } @@ -153,7 +153,7 @@ TEST_F(BtreeTests, insert_does_not_insert_imaginary_values) tree->insert(key, value); l = tree->lookup(key); - ASSERT_TRUE(l); + ASSERT_TRUE(!!l); ASSERT_THAT(*l, Eq(100u)); key[0] = 1; @@ -183,7 +183,7 @@ TEST_F(BtreeTests, clone) uint64_t value = i * 7; l = tree->lookup(key); - ASSERT_TRUE(l); + ASSERT_TRUE(!!l); ASSERT_THAT(*l, Eq(value)); } @@ -200,11 +200,11 @@ TEST_F(BtreeTests, clone) uint64_t value = i * 7; l = tree->lookup(key); - ASSERT_TRUE(l); + ASSERT_TRUE(!!l); ASSERT_THAT(*l, Eq(value)); l = copy->lookup(key); - ASSERT_TRUE(l); + ASSERT_TRUE(!!l); ASSERT_THAT(*l, Eq(value)); } @@ -216,7 +216,7 @@ TEST_F(BtreeTests, clone) ASSERT_FALSE(l); l = copy->lookup(key); - ASSERT_TRUE(l); + ASSERT_TRUE(!!l); ASSERT_THAT(*l, Eq(value)); } diff --git a/unit-tests/run_set_t.cc b/unit-tests/run_set_t.cc index abe201c..53d1bc1 100644 --- a/unit-tests/run_set_t.cc +++ b/unit-tests/run_set_t.cc @@ -36,7 +36,7 @@ namespace { TEST_F(RunSetTests, create) { - auto_ptr > rs(new run_set()); + unique_ptr > rs(new run_set()); } TEST_F(RunSetTests, add_single_blocks) diff --git a/unit-tests/space_map_t.cc b/unit-tests/space_map_t.cc index 0848909..d11de47 100644 --- a/unit-tests/space_map_t.cc +++ b/unit-tests/space_map_t.cc @@ -101,7 +101,7 @@ namespace { for (unsigned i = 0; i < NR_BLOCKS; i++) { boost::optional mb = sm->new_block(); - ASSERT_TRUE(mb); + ASSERT_TRUE(!!mb); ASSERT_THAT(sm->get_nr_free(), Eq(NR_BLOCKS - i - 1)); } @@ -137,7 +137,7 @@ namespace { void test_not_allocated_twice(space_map::ptr sm) { boost::optional mb = sm->new_block(); - ASSERT_TRUE(mb); + ASSERT_TRUE(!!mb); for (;;) { boost::optional b = sm->new_block(); diff --git a/unit-tests/test_utils.h b/unit-tests/test_utils.h index b7d32c2..bdd63e5 100644 --- a/unit-tests/test_utils.h +++ b/unit-tests/test_utils.h @@ -111,7 +111,7 @@ namespace test { throw std::runtime_error("system failed"); } - std::auto_ptr dir_; + std::unique_ptr dir_; }; //-------------------------------- From 0d510924d552cc7e3ccdc5ec0dff980e354813da Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 10:42:42 +0000 Subject: [PATCH 031/118] [thin_delta] set nr_data_blocks to zero if there are no space maps. ie. a metadata_snap is being used. --- thin-provisioning/thin_delta.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 3ec71c8..3b33c73 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -527,7 +527,7 @@ namespace local { mapping_recorder mr2; damage_visitor damage_v; superblock_detail::superblock sb; - checked_space_map::ptr data_sm; + block_address nr_data_blocks = 0ull; { block_manager<>::ptr bm = open_bm(*fs.dev, block_manager<>::READ_ONLY, !fs.use_metadata_snap); @@ -562,13 +562,16 @@ namespace local { btree_visit_values(snap2, mr2, damage_v); mr2.complete(); + + if (md->data_sm_) + nr_data_blocks = md->data_sm_->get_nr_blocks(); } indented_stream is(cout); begin_superblock(is, "", sb.time_, sb.trans_id_, sb.data_block_size_, - data_sm->get_nr_blocks(), + nr_data_blocks, sb.metadata_snap_ ? boost::optional(sb.metadata_snap_) : boost::optional()); From 7c2b3fb67158668e0f05ccefec428775057a8931 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 11:44:44 +0000 Subject: [PATCH 032/118] [thin_delta] Fix bug when comparing the mappings --- thin-provisioning/thin_delta.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 3b33c73..61f910f 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -432,7 +432,8 @@ namespace local { mapping left_mapping; mapping right_mapping; - while (left_it != left.end() && right_it != right.end()) { + while ((left_mapping.len_ || left_it != left.end()) && + (right_mapping.len_ || right_it != right.end())) { if (!left_mapping.len_ && left_it != left.end()) left_mapping = *left_it++; From 1f10017635e84246b1ca9dcc20d7c71c95d346d2 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 15:17:02 +0000 Subject: [PATCH 033/118] [thin_delta] tidy up the comparison function --- thin-provisioning/thin_delta.cc | 123 +++++++++++++++++++------------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 61f910f..9d6337a 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -103,13 +103,6 @@ namespace local { len_(len) { } - void consume(uint64_t delta) { - delta = min(delta, len_); - vbegin_ += delta; - dbegin_ += delta; - len_ -= delta; - } - uint64_t vbegin_, dbegin_, len_; }; @@ -120,6 +113,51 @@ namespace local { return out; } + //-------------------------------- + + template + class mapping_stream { + public: + mapping_stream(Container const &c) + : it_(c.begin()), + end_(c.end()) { + m_ = *it_; + } + + mapping const &get_mapping() const { + return m_; + } + + bool more_mappings() const { + return it_ != end_; + } + + void consume(uint64_t delta) { + if (it_ == end_) + throw runtime_error("end of stream already reached"); + + if (delta > m_.len_) + throw runtime_error("delta too long"); + + if (delta == m_.len_) { + ++it_; + m_ = *it_; + + } else { + m_.vbegin_ += delta; + m_.dbegin_ += delta; + m_.len_ -= delta; + } + } + + private: + typename Container::const_iterator it_; + typename Container::const_iterator end_; + mapping m_; + }; + + //-------------------------------- + typedef std::deque mapping_deque; // Builds up an in core rep of the mappings for a device. @@ -426,58 +464,45 @@ namespace local { // We iterate through both sets of mappings in parallel // noting any differences. - mapping_deque::const_iterator left_it = left.begin(); - mapping_deque::const_iterator right_it = right.begin(); + mapping_stream ls{left}; + mapping_stream rs{right}; - mapping left_mapping; - mapping right_mapping; + while (ls.more_mappings() && rs.more_mappings()) { + auto &lm = ls.get_mapping(); + auto &rm = rs.get_mapping(); - while ((left_mapping.len_ || left_it != left.end()) && - (right_mapping.len_ || right_it != right.end())) { - if (!left_mapping.len_ && left_it != left.end()) - left_mapping = *left_it++; + if (lm.vbegin_ < rm.vbegin_) { + auto delta = min(lm.len_, rm.vbegin_ - lm.vbegin_); + e.left_only(lm.vbegin_, lm.dbegin_, delta); + ls.consume(delta); - if (!right_mapping.len_ && right_it != right.end()) - right_mapping = *right_it++; + } else if (lm.vbegin_ > rm.vbegin_) { + auto delta = min(rm.len_, lm.vbegin_ - rm.vbegin_); + e.right_only(rm.vbegin_, rm.dbegin_, delta); + rs.consume(delta); - while (left_mapping.len_ && right_mapping.len_) { - if (left_mapping.vbegin_ < right_mapping.vbegin_) { - uint64_t delta = min(left_mapping.len_, right_mapping.vbegin_ - left_mapping.vbegin_); - e.left_only(left_mapping.vbegin_, left_mapping.dbegin_, delta); - left_mapping.consume(delta); + } else if (lm.dbegin_ != rm.dbegin_) { + auto delta = min(lm.len_, rm.len_); + e.blocks_differ(lm.vbegin_, lm.dbegin_, rm.dbegin_, delta); + ls.consume(delta); + rs.consume(delta); - } else if (left_mapping.vbegin_ > right_mapping.vbegin_) { - uint64_t delta = min(right_mapping.len_, left_mapping.vbegin_ - right_mapping.vbegin_); - e.right_only(right_mapping.vbegin_, right_mapping.dbegin_, delta); - right_mapping.consume(delta); - - } else if (left_mapping.dbegin_ != right_mapping.dbegin_) { - uint64_t delta = min(left_mapping.len_, right_mapping.len_); - e.blocks_differ(left_mapping.vbegin_, left_mapping.dbegin_, right_mapping.dbegin_, delta); - left_mapping.consume(delta); - right_mapping.consume(delta); - - } else { - uint64_t delta = min(left_mapping.len_, right_mapping.len_); - e.blocks_same(left_mapping.vbegin_, left_mapping.dbegin_, delta); - left_mapping.consume(delta); - right_mapping.consume(delta); - } + } else { + auto delta = min(lm.len_, rm.len_); + e.blocks_same(lm.vbegin_, lm.dbegin_, delta); + ls.consume(delta); + rs.consume(delta); } } - while (left_it != left.end()) { - left_mapping = *left_it++; - - if (left_mapping.len_) - e.left_only(left_mapping.vbegin_, left_mapping.dbegin_, left_mapping.len_); + while (ls.more_mappings()) { + auto &lm = ls.get_mapping(); + e.left_only(lm.vbegin_, lm.dbegin_, lm.len_); } - while (right_it != right.end()) { - right_mapping = *right_it++; - - if (right_mapping.len_) - e.right_only(right_mapping.vbegin_, right_mapping.dbegin_, right_mapping.len_); + while (rs.more_mappings()) { + auto &rm = rs.get_mapping(); + e.right_only(rm.vbegin_, rm.dbegin_, rm.len_); } e.complete(); From f193a70a310a75f363cdd5c5b1e09712e8d71b83 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 18 Feb 2016 11:31:43 +0000 Subject: [PATCH 034/118] [thin-delta] fix bug in earlier refactor --- thin-provisioning/thin_delta.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 9d6337a..346d62c 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -498,11 +498,13 @@ namespace local { while (ls.more_mappings()) { auto &lm = ls.get_mapping(); e.left_only(lm.vbegin_, lm.dbegin_, lm.len_); + ls.consume(lm.len_); } while (rs.more_mappings()) { auto &rm = rs.get_mapping(); e.right_only(rm.vbegin_, rm.dbegin_, rm.len_); + rs.consume(rm.len_); } e.complete(); From 5f879237f5738ec1a66a5ab80eb60bda12256030 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 24 Feb 2016 13:41:43 +0000 Subject: [PATCH 035/118] [build] add --enable-dev-tools configure flag This builds tools that I want to keep in the same code base, but not intended for general release. --- Makefile.in | 14 +++++++++++- configure.ac | 9 ++++++++ thin-provisioning/commands.cc | 4 ++++ thin-provisioning/commands.h | 7 ++++++ thin-provisioning/thin_generate_metadata.cc | 25 +++++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 thin-provisioning/thin_generate_metadata.cc diff --git a/Makefile.in b/Makefile.in index da0e5c7..d4ff973 100644 --- a/Makefile.in +++ b/Makefile.in @@ -95,7 +95,14 @@ SOURCE=\ thin-provisioning/thin_restore.cc \ thin-provisioning/thin_rmap.cc \ thin-provisioning/thin_trim.cc \ - thin-provisioning/xml_format.cc + thin-provisioning/xml_format.cc \ + +DEVTOOLS_SOURCE=\ + thin-provisioning/thin_generate_metadata.cc + +ifeq ("@DEVTOOLS@", "yes") +SOURCE+=$(DEVTOOLS_SOURCE) +endif CC:=@CC@ CXX:=@CXX@ @@ -106,6 +113,11 @@ TOP_BUILDDIR:=@top_builddir@ CFLAGS+=-g -Wall -O3 CFLAGS+=@LFS_FLAGS@ CXXFLAGS+=-g -Wall -fno-strict-aliasing -std=c++11 + +ifeq ("@DEVTOOLS@", "yes") +CXXFLAGS+=-DDEV_TOOLS +endif + CXXFLAGS+=@CXXOPTIMISE_FLAG@ CXXFLAGS+=@CXXDEBUG_FLAG@ CXXFLAGS+=@CXX_STRERROR_FLAG@ diff --git a/configure.ac b/configure.ac index 1161145..d9600b9 100644 --- a/configure.ac +++ b/configure.ac @@ -137,6 +137,14 @@ AC_ARG_ENABLE(testing, TESTING=$enableval, TESTING=no) AC_MSG_RESULT($TESTING) +################################################################################ +dnl -- Enable development tools +AC_MSG_CHECKING(whenter to enable development tools) +AC_ARG_ENABLE(dev-tools, + AC_HELP_STRING(--enable-dev-tools, [enable development tools in the makefile]), + DEVTOOLS=$enableval, DEVTOOLS=no) +AC_MSG_RESULT($DEVTOOLS) + ################################################################################ dnl -- Enable static libstdc++ AC_MSG_CHECKING(whether to statically link libstdc++) @@ -173,6 +181,7 @@ AC_SUBST(RELEASE_DATE) AC_SUBST(TESTING) AC_SUBST(THIN_PROVISIONING_TOOLS_VERSION) AC_SUBST(STATIC_CXX) +AC_SUBST(DEVTOOLS) ################################################################################ dnl -- First and last lines should not contain files to generate in order to diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index c25abce..62b2c60 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -17,6 +17,10 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_repair_cmd())); app.add_cmd(command::ptr(new thin_rmap_cmd())); app.add_cmd(command::ptr(new thin_trim_cmd())); + +#if DEV_COMMANDS + app.add_cmd(command::ptr(new thin_generate_metadata_cmd())); +#endif } //---------------------------------------------------------------- diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index ec1f1ec..4ff97ea 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -70,6 +70,13 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_generate_metadata_cmd : public base::command { + public: + thin_generate_metadata_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + void register_thin_commands(base::application &app); } diff --git a/thin-provisioning/thin_generate_metadata.cc b/thin-provisioning/thin_generate_metadata.cc new file mode 100644 index 0000000..4d5903d --- /dev/null +++ b/thin-provisioning/thin_generate_metadata.cc @@ -0,0 +1,25 @@ +#include "thin-provisioning/commands.h" + +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +thin_generate_metadata_cmd::thin_generate_metadata_cmd() + : command("thin_generate_metadata") +{ +} + +void +thin_generate_metadata_cmd::usage(std::ostream &out) const +{ + +} + +int +thin_generate_metadata_cmd::run(int argc, char **argv) +{ + + return 1; +} + +//---------------------------------------------------------------- From 6637a3061852999e63d6b59a7c3905f226aa4c47 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 24 Feb 2016 14:42:37 +0000 Subject: [PATCH 036/118] [base] Move container_of to own file --- base/container_of.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 base/container_of.h diff --git a/base/container_of.h b/base/container_of.h new file mode 100644 index 0000000..1d70f31 --- /dev/null +++ b/base/container_of.h @@ -0,0 +1,24 @@ +#ifndef BASE_CONTAINER_OF_H +#define BASE_CONTAINER_OF_H + +#include + +//---------------------------------------------------------------- + +namespace base { + template + size_t offsetof__(const M P::*member) + { + return (size_t) &( reinterpret_cast(0)->*member); + } + + template + P *container_of(M *ptr, M const P::*member) + { + return (P *)((char *)(ptr) - offsetof__(member)); + } +} + +//---------------------------------------------------------------- + +#endif From c4215c0cf6ea1088b2085d9c9d483f5fed2aa893 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 24 Feb 2016 15:32:05 +0000 Subject: [PATCH 037/118] [thin_show_metadata] thin_show_blocks.cc -> thin_show_metadata.cc --- Makefile.in | 2 +- .../{thin_show_blocks.cc => thin_show_metadata.cc} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename thin-provisioning/{thin_show_blocks.cc => thin_show_metadata.cc} (100%) diff --git a/Makefile.in b/Makefile.in index 0365e1e..11267cc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -106,7 +106,7 @@ SOURCE=\ DEVTOOLS_SOURCE=\ thin-provisioning/thin_generate_metadata.cc \ thin-provisioning/variable_chunk_stream.cc \ - thin-provisioning/thin_show_blocks.cc + thin-provisioning/thin_show_metadata.cc ifeq ("@DEVTOOLS@", "yes") SOURCE+=$(DEVTOOLS_SOURCE) diff --git a/thin-provisioning/thin_show_blocks.cc b/thin-provisioning/thin_show_metadata.cc similarity index 100% rename from thin-provisioning/thin_show_blocks.cc rename to thin-provisioning/thin_show_metadata.cc From e78de5d3ad8b449e40e47041d4bacc4cfb1bd3f6 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 26 Feb 2016 12:50:17 +0000 Subject: [PATCH 038/118] [thin_show_metadata] ncurses render --- Makefile.in | 7 +- VERSION | 2 +- thin-provisioning/superblock.cc | 14 +- thin-provisioning/superblock.h | 2 + thin-provisioning/thin_show_metadata.cc | 343 +++++++++++++++++++----- ui/ui.cc | 36 +++ ui/ui.h | 20 ++ 7 files changed, 358 insertions(+), 66 deletions(-) create mode 100644 ui/ui.cc create mode 100644 ui/ui.h diff --git a/Makefile.in b/Makefile.in index 11267cc..8067de9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -106,7 +106,8 @@ SOURCE=\ DEVTOOLS_SOURCE=\ thin-provisioning/thin_generate_metadata.cc \ thin-provisioning/variable_chunk_stream.cc \ - thin-provisioning/thin_show_metadata.cc + thin-provisioning/thin_show_metadata.cc \ + ui/ui.cc ifeq ("@DEVTOOLS@", "yes") SOURCE+=$(DEVTOOLS_SOURCE) @@ -133,6 +134,10 @@ CXXFLAGS+=@LFS_FLAGS@ INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning LIBS:=-laio -lexpat +ifeq ("@DEVTOOLS@", "yes") +LIBS+=-lncurses +endif + ifeq ("@STATIC_CXX@", "yes") CXXLIB+=-Wl,-Bstatic -lstdc++ -Wl,-Bdynamic -Wl,--as-needed else diff --git a/VERSION b/VERSION index e6d527c..e1bde80 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.2-rc3 +0.7.0-dev diff --git a/thin-provisioning/superblock.cc b/thin-provisioning/superblock.cc index 9258b53..4f4f844 100644 --- a/thin-provisioning/superblock.cc +++ b/thin-provisioning/superblock.cc @@ -146,22 +146,32 @@ namespace thin_provisioning { } } - superblock_detail::superblock read_superblock(block_manager<>::ptr bm, block_address location) + superblock_detail::superblock read_superblock(block_manager<> const &bm, block_address location) { using namespace superblock_detail; superblock sb; - block_manager<>::read_ref r = bm->read_lock(location, superblock_validator()); + block_manager<>::read_ref r = bm.read_lock(location, superblock_validator()); superblock_disk const *sbd = reinterpret_cast(r.data()); superblock_traits::unpack(*sbd, sb); return sb; } + superblock_detail::superblock read_superblock(block_manager<>::ptr bm, block_address location) + { + return read_superblock(*bm, location); + } + superblock_detail::superblock read_superblock(block_manager<>::ptr bm) { return read_superblock(bm, SUPERBLOCK_LOCATION); } + superblock_detail::superblock read_superblock(block_manager<> const &bm) + { + return read_superblock(bm, SUPERBLOCK_LOCATION); + } + void write_superblock(block_manager<>::ptr bm, superblock_detail::superblock const &sb) { block_manager<>::write_ref w = bm->write_lock(SUPERBLOCK_LOCATION, superblock_validator()); diff --git a/thin-provisioning/superblock.h b/thin-provisioning/superblock.h index f527a15..5e1ba2c 100644 --- a/thin-provisioning/superblock.h +++ b/thin-provisioning/superblock.h @@ -128,6 +128,8 @@ namespace thin_provisioning { // FIXME: should we put init_superblock in here too? + // FIXME: make the bm const, and pass by reference rather than ptr + superblock_detail::superblock read_superblock(persistent_data::block_manager<> const &bm); superblock_detail::superblock read_superblock(persistent_data::block_manager<>::ptr bm); superblock_detail::superblock read_superblock(persistent_data::block_manager<>::ptr bm, persistent_data::block_address location); diff --git a/thin-provisioning/thin_show_metadata.cc b/thin-provisioning/thin_show_metadata.cc index 2a7140a..5d181c0 100644 --- a/thin-provisioning/thin_show_metadata.cc +++ b/thin-provisioning/thin_show_metadata.cc @@ -5,6 +5,7 @@ #include "thin-provisioning/commands.h" #include "thin-provisioning/metadata.h" #include "thin-provisioning/superblock.h" +#include "ui/ui.h" #include "version.h" #include @@ -12,82 +13,282 @@ #include #include #include +#include using namespace persistent_data; using namespace sm_disk_detail; using namespace std; using namespace thin_provisioning; +using namespace ui; //---------------------------------------------------------------- namespace { - bool is_superblock(block_manager<>::read_ref &rr) { - using namespace superblock_detail; + class examiner { + public: + examiner(string const &name, int colour_pair, char rep) + : name_(name), + colour_pair_(colour_pair), + rep_(rep) { + } - superblock_disk const *sbd = reinterpret_cast(rr.data()); - if (to_cpu(sbd->magic_) == SUPERBLOCK_MAGIC) { - superblock sb; - superblock_traits::unpack(*sbd, sb); - cout << "metadata nr blocks: " << sb.metadata_nr_blocks_ << endl; + virtual ~examiner() {} + virtual bool recognise(block_manager<>::read_ref rr) const = 0; +// virtual void render_block(text_ui &ui, block_manager<>::read_ref rr) = 0; + + string const &get_name() const { + return name_; + } + + int get_color_pair() const { + return colour_pair_; + } + + char get_rep() const { + return rep_; + } + + private: + string name_; + int colour_pair_; + char rep_; + }; + + class raw_examiner : public examiner { + public: + raw_examiner() + : examiner("raw", 5, '?') { + } + + virtual bool recognise(block_manager<>::read_ref rr) const { return true; } + }; - return false; - } - - bool is_bitmap_block(block_manager<>::read_ref &rr) { - bitmap_header const *data = reinterpret_cast(rr.data()); - crc32c sum(BITMAP_CSUM_XOR); - sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); - return sum.get_sum() == to_cpu(data->csum); - } - - bool is_index_block(block_manager<>::read_ref &rr) { - metadata_index const *mi = reinterpret_cast(rr.data()); - crc32c sum(INDEX_CSUM_XOR); - sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t)); - return sum.get_sum() == to_cpu(mi->csum_); - } - - bool is_btree_node(block_manager<>::read_ref &rr) { - using namespace btree_detail; - - disk_node const *data = reinterpret_cast(rr.data()); - node_header const *n = &data->header; - crc32c sum(BTREE_CSUM_XOR); - sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); - return sum.get_sum() == to_cpu(n->csum); - } - - void show_blocks(string const &dev) { - block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY); - - metadata md(bm, metadata::OPEN); - cout << "Metadata space map: nr_blocks = " << md.metadata_sm_->get_nr_blocks() - << ", nr_free_blocks = " << md.metadata_sm_->get_nr_free() - << endl; - cout << "Data space map: nr_blocks = " << md.data_sm_->get_nr_blocks() - << ", nr_free_blocks = " << md.data_sm_->get_nr_free() - << endl; - - block_address nr_blocks = bm->get_nr_blocks(); - for (block_address b = 0; b < nr_blocks; b++) { - block_manager<>::read_ref rr = bm->read_lock(b); - - if (is_superblock(rr)) - cout << b << ": superblock" << endl; - - else if (is_bitmap_block(rr)) - cout << b << ": bitmap block" << endl; - - else if (is_btree_node(rr)) - cout << b << ": btree_node" << endl; - - else - cout << b << ": unknown" << endl; + class superblock_examiner : public examiner { + public: + superblock_examiner() + : examiner("superblock", 1, 'S') { } - } + + virtual bool recognise(block_manager<>::read_ref rr) const { + using namespace superblock_detail; + + superblock_disk const *sbd = reinterpret_cast(rr.data()); + if (to_cpu(sbd->magic_) == SUPERBLOCK_MAGIC) { + superblock sb; + superblock_traits::unpack(*sbd, sb); + cout << "metadata nr blocks: " << sb.metadata_nr_blocks_ << endl; + + return true; + } + + return false; + } + }; + + class bitmap_examiner : public examiner { + public: + bitmap_examiner() + : examiner("bitmap", 2, ':') { + } + + virtual bool recognise(block_manager<>::read_ref rr) const { + bitmap_header const *data = reinterpret_cast(rr.data()); + crc32c sum(BITMAP_CSUM_XOR); + sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); + return sum.get_sum() == to_cpu(data->csum); + } + }; + + class index_examiner : public examiner { + public: + index_examiner() + : examiner("index", 3, 'i') { + } + + virtual bool recognise(block_manager<>::read_ref rr) const { + metadata_index const *mi = reinterpret_cast(rr.data()); + crc32c sum(INDEX_CSUM_XOR); + sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t)); + return sum.get_sum() == to_cpu(mi->csum_); + } + }; + + + class btree_examiner : public examiner { + public: + btree_examiner(string const &name, int colour_pair, char c) + : examiner(name, colour_pair, c) { + } + + bool is_btree_node(block_manager<>::read_ref rr) const { + using namespace btree_detail; + + disk_node const *data = reinterpret_cast(rr.data()); + node_header const *n = &data->header; + crc32c sum(BTREE_CSUM_XOR); + sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + return sum.get_sum() == to_cpu(n->csum); + } + }; + + class dev_detail_examiner : public btree_examiner { + public: + dev_detail_examiner() + : btree_examiner("dev_details", 4, 'd') { + } + + virtual bool recognise(block_manager<>::read_ref rr) const { + if (!btree_examiner::is_btree_node(rr)) + return false; + + using namespace btree_detail; + + disk_node const *data = reinterpret_cast(rr.data()); + node_header const *n = &data->header; + return to_cpu(n->value_size) == sizeof(device_tree_detail::device_details_disk); + } + }; + + class ref_count_examiner : public btree_examiner { + public: + ref_count_examiner() + : btree_examiner("ref_count node", 6, 'r') { + } + + virtual bool recognise(block_manager<>::read_ref rr) const { + if (!btree_examiner::is_btree_node(rr)) + return false; + + using namespace btree_detail; + + disk_node const *data = reinterpret_cast(rr.data()); + node_header const *n = &data->header; + return to_cpu(n->value_size) == sizeof(uint32_t); + } + }; + + class mapping_examiner : public btree_examiner { + public: + mapping_examiner() + : btree_examiner("mapping node", 7, 'm') { + } + + virtual bool recognise(block_manager<>::read_ref rr) const { + if (!btree_examiner::is_btree_node(rr)) + return false; + + using namespace btree_detail; + + disk_node const *data = reinterpret_cast(rr.data()); + node_header const *n = &data->header; + return to_cpu(n->value_size) == sizeof(uint64_t); + } + }; + + class main_dialog { + public: + main_dialog(text_ui &ui, + block_manager<> const &bm) + : ui_(ui), + bm_(bm), + raw_examiner_(new raw_examiner()) { + + examiners_.push_back(shared_ptr(new superblock_examiner())); + examiners_.push_back(shared_ptr(new bitmap_examiner())); + examiners_.push_back(shared_ptr(new index_examiner())); + examiners_.push_back(shared_ptr(new dev_detail_examiner())); + examiners_.push_back(shared_ptr(new ref_count_examiner())); + examiners_.push_back(shared_ptr(new mapping_examiner())); + } + + void run() { + auto line_length = 80; + for (block_address b = 0; b < 2000; b++) { + block_manager<>::read_ref rr = bm_.read_lock(b); + + if (!(b % line_length)) { + if (b > 0) + printw("\n"); + + printw("%8llu: ", b); + } + + auto e = find_examiner(rr); + attron(COLOR_PAIR(e->get_color_pair())); + printw("%c", e->get_rep()); + attroff(COLOR_PAIR(e->get_color_pair())); + } + + printw("\n"); + show_superblock(); + } + + private: + void show_superblock() { + auto sb = read_superblock(bm_); + + printw("\n\nSuperblock at 0\n"); + printw("data mapping root: %llu\n", sb.data_mapping_root_); + printw("device details root: %llu\n", sb.device_details_root_); + printw("data block size: %u\n", sb.data_block_size_); + printw("metadata nr blocks: %llu\n", sb.metadata_nr_blocks_); + } + + shared_ptr &find_examiner(block_manager<>::read_ref const &rr) { + for (shared_ptr &e : examiners_) { + if (e->recognise(rr)) + return e; + } + + return raw_examiner_; + } + + text_ui &ui_; + block_manager<> const &bm_; + list> examiners_; + shared_ptr raw_examiner_; + + +#if 0 + void show_superblock(text_ui &ui, superblock_detail::superblock const &sb) { + } + + void show_blocks(text_ui &ui, string const &dev) { + metadata md(bm); + + show_superblock(ui, md.sb_); + +#if 0 + cout << "Metadata space map: nr_blocks = " << md.metadata_sm_->get_nr_blocks() + << ", nr_free_blocks = " << md.metadata_sm_->get_nr_free() + << endl; + cout << "Data space map: nr_blocks = " << md.data_sm_->get_nr_blocks() + << ", nr_free_blocks = " << md.data_sm_->get_nr_free() + << endl; + + block_address nr_blocks = bm->get_nr_blocks(); + for (block_address b = 0; b < nr_blocks; b++) { + block_manager<>::read_ref rr = bm->read_lock(b); + + if (is_superblock(rr)) + cout << b << ": superblock" << endl; + + else if (is_bitmap_block(rr)) + cout << b << ": bitmap block" << endl; + + else if (is_btree_node(rr)) + cout << b << ": btree_node" << endl; + + else + cout << b << ": unknown" << endl; + } +#endif + } +#endif + }; } //---------------------------------------------------------------- @@ -135,7 +336,25 @@ thin_show_metadata_cmd::run(int argc, char **argv) } try { - show_blocks(argv[optind]); + ui::text_ui ui; + + block_manager<>::ptr bm = open_bm(argv[optind], block_manager<>::READ_ONLY, true); + main_dialog dialog(ui, *bm); + dialog.run(); +#if 0 +// show_blocks(ui, argv[optind]); +#endif + + +#if 0 + attron(COLOR_PAIR(1)); + printw("Hello, "); + attron(A_BOLD); + printw("world!\n"); + attroff(A_BOLD); + attroff(COLOR_PAIR(1)); +#endif + getch(); } catch (std::exception const &e) { cerr << e.what() << endl; diff --git a/ui/ui.cc b/ui/ui.cc new file mode 100644 index 0000000..2df3d6e --- /dev/null +++ b/ui/ui.cc @@ -0,0 +1,36 @@ +#include "ui/ui.h" + +#include + +using namespace ui; + +//---------------------------------------------------------------- + +text_ui::text_ui() +{ + initscr(); + noecho(); + + start_color(); + init_pair(1, COLOR_RED, COLOR_BLACK); + init_pair(2, COLOR_YELLOW, COLOR_BLACK); + init_pair(3, COLOR_BLUE, COLOR_BLACK); + init_pair(4, COLOR_GREEN, COLOR_BLACK); + init_pair(5, COLOR_YELLOW, COLOR_BLACK); + init_pair(6, COLOR_BLACK, COLOR_RED); + init_pair(7, COLOR_WHITE, COLOR_BLACK); + +} + +text_ui::~text_ui() +{ + endwin(); +} + +void +text_ui::refresh() +{ + refresh(); +} + +//---------------------------------------------------------------- diff --git a/ui/ui.h b/ui/ui.h new file mode 100644 index 0000000..75a2808 --- /dev/null +++ b/ui/ui.h @@ -0,0 +1,20 @@ +#ifndef UI_UI_H + +#include + +//---------------------------------------------------------------- + +namespace ui { + class text_ui { + public: + text_ui(); + ~text_ui(); + + void refresh(); + }; +}; + + +//---------------------------------------------------------------- + +#endif From 11cd796652523f4c6a4d23fd121ee0dac34a9f25 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:27:27 +0800 Subject: [PATCH 039/118] Show the block address if block_cache::get() failed --- block-cache/block_cache.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/block-cache/block_cache.cc b/block-cache/block_cache.cc index 6ecce1f..e7e6f14 100644 --- a/block-cache/block_cache.cc +++ b/block-cache/block_cache.cc @@ -620,7 +620,9 @@ block_cache::get(block_address index, unsigned flags, validator::ptr v) return *b; } - throw std::runtime_error("couldn't get block"); + std::ostringstream out; + out << "couldn't get block " << index; + throw std::runtime_error(out.str()); } void From b1d4b9f7c8e9953b5222278a137fc6b115444364 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:28:41 +0800 Subject: [PATCH 040/118] Show the block address of error nodes --- .../data-structures/btree_damage_visitor.h | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index fa11378..f0368bb 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -300,14 +300,16 @@ namespace persistent_data { size_t elt_size = sizeof(uint64_t) + n.get_value_size(); if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { std::ostringstream out; - out << "max entries too large: " << n.get_max_entries(); + out << "max entries too large: " << n.get_max_entries() + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } if (n.get_max_entries() % 3) { std::ostringstream out; - out << "max entries is not divisible by 3: " << n.get_max_entries(); + out << "max entries is not divisible by 3: " << n.get_max_entries() + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -321,7 +323,8 @@ namespace persistent_data { std::ostringstream out; out << "bad nr_entries: " << n.get_nr_entries() << " < " - << n.get_max_entries(); + << n.get_max_entries() + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -333,7 +336,8 @@ namespace persistent_data { << n.get_nr_entries() << ", expected at least " << min - << "(max_entries = " << n.get_max_entries() << ")"; + << " (block " << n.get_location() + << ", max_entries = " << n.get_max_entries() << ")"; report_damage(out.str()); return false; } @@ -354,7 +358,8 @@ namespace persistent_data { uint64_t k = n.key_at(i); if (k <= last_key) { ostringstream out; - out << "keys are out of order, " << k << " <= " << last_key; + out << "keys are out of order, " << k << " <= " << last_key + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -372,7 +377,8 @@ namespace persistent_data { if (*key > n.key_at(0)) { ostringstream out; out << "parent key mismatch: parent was " << *key - << ", but lowest in node was " << n.key_at(0); + << ", but lowest in node was " << n.key_at(0) + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -388,7 +394,8 @@ namespace persistent_data { if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) { ostringstream out; out << "the last key of the previous leaf was " << *last_leaf_key_[level] - << " and the first key of this leaf is " << n.key_at(0); + << " and the first key of this leaf is " << n.key_at(0) + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } From d2260dee34f80844f6a2468c753d0d2a239f47c1 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:30:45 +0800 Subject: [PATCH 041/118] Show the block address in exception string --- persistent-data/data-structures/btree.tcc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/persistent-data/data-structures/btree.tcc b/persistent-data/data-structures/btree.tcc index 80f2b94..06d0deb 100644 --- a/persistent-data/data-structures/btree.tcc +++ b/persistent-data/data-structures/btree.tcc @@ -90,14 +90,22 @@ namespace persistent_data { { uint32_t flags = to_cpu(raw_->header.flags); if (flags & INTERNAL_NODE) { - if (flags & LEAF_NODE) - throw runtime_error("btree node is both internal and leaf"); + if (flags & LEAF_NODE) { + ostringstream out; + out << "btree node is both internal and leaf" + << " (block " << location_ << ")"; + throw runtime_error(out.str()); + } return INTERNAL; } else if (flags & LEAF_NODE) return LEAF; - else - throw runtime_error("unknown node type"); + else { + ostringstream out; + out << "unknown node type" + << " (block " << location_ << ")"; + throw runtime_error(out.str()); + } } template @@ -352,7 +360,8 @@ namespace persistent_data { std::ostringstream out; out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type) << ", but got " << get_value_size() - << ". This is not the btree you are looking for." << std::endl; + << ". This is not the btree you are looking for." + << " (block " << location_ << ")" << std::endl; return out.str(); } @@ -371,7 +380,8 @@ namespace persistent_data { if (max < get_nr_entries()) { std::ostringstream out; out << "Bad nr of elements: max per block = " - << max << ", actual = " << get_nr_entries() << std::endl; + << max << ", actual = " << get_nr_entries() + << " (block " << location_ << ")" << std::endl; throw std::runtime_error(out.str()); } From b47c02ed8bbd1623a5db9936b33f6d80306d53b9 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:31:24 +0800 Subject: [PATCH 042/118] Show the wanted checksum in bad-superblock-checksum exception --- thin-provisioning/superblock.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/superblock.cc b/thin-provisioning/superblock.cc index 9258b53..fb0d17e 100644 --- a/thin-provisioning/superblock.cc +++ b/thin-provisioning/superblock.cc @@ -91,8 +91,11 @@ namespace { superblock_disk const *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); - if (sum.get_sum() != to_cpu(sbd->csum_)) - throw checksum_error("bad checksum in superblock"); + if (sum.get_sum() != to_cpu(sbd->csum_)) { + ostringstream out; + out << "bad checksum in superblock, wanted " << sum.get_sum(); + throw checksum_error(out.str()); + } } virtual void prepare(void *raw, block_address location) const { From 7ec47158b56eacf2d9642997adb13adca6638a84 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:02 +0800 Subject: [PATCH 043/118] [emitter] Preserve the flags and version entries in superblock --- thin-provisioning/emitter.h | 2 ++ thin-provisioning/human_readable_format.cc | 4 ++++ thin-provisioning/metadata_dumper.cc | 2 ++ thin-provisioning/restore_emitter.cc | 4 ++++ thin-provisioning/xml_format.cc | 6 ++++++ 5 files changed, 18 insertions(+) diff --git a/thin-provisioning/emitter.h b/thin-provisioning/emitter.h index 58658a9..bb944da 100644 --- a/thin-provisioning/emitter.h +++ b/thin-provisioning/emitter.h @@ -49,6 +49,8 @@ namespace thin_provisioning { virtual void begin_superblock(std::string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) = 0; diff --git a/thin-provisioning/human_readable_format.cc b/thin-provisioning/human_readable_format.cc index 3cfc188..6c53b3c 100644 --- a/thin-provisioning/human_readable_format.cc +++ b/thin-provisioning/human_readable_format.cc @@ -43,12 +43,16 @@ namespace { void begin_superblock(string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) { out_ << "begin superblock: \"" << uuid << "\"" << ", " << time << ", " << trans_id + << ", " << (flags ? *flags : 0) + << ", " << (version ? *version : 1) << ", " << data_block_size << ", " << nr_data_blocks; if (metadata_snap) diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index 0c0fbe4..5006fc0 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -228,6 +228,8 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) e->begin_superblock("", md->sb_.time_, md->sb_.trans_id_, + md->sb_.flags_, + md->sb_.version_, md->sb_.data_block_size_, nr_data_blocks, boost::optional()); diff --git a/thin-provisioning/restore_emitter.cc b/thin-provisioning/restore_emitter.cc index fd83f56..2fab704 100644 --- a/thin-provisioning/restore_emitter.cc +++ b/thin-provisioning/restore_emitter.cc @@ -44,6 +44,8 @@ namespace { virtual void begin_superblock(std::string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) { @@ -54,6 +56,8 @@ namespace { memcpy(&sb.uuid_, uuid.c_str(), std::min(sizeof(sb.uuid_), uuid.length())); sb.time_ = time; sb.trans_id_ = trans_id; + sb.flags_ = flags ? *flags : 0; + sb.version_ = version ? *version : 1; sb.data_block_size_ = data_block_size; sb.metadata_snap_ = metadata_snap ? *metadata_snap : 0; md_->data_sm_->extend(nr_data_blocks); diff --git a/thin-provisioning/xml_format.cc b/thin-provisioning/xml_format.cc index 333204f..efbfd3f 100644 --- a/thin-provisioning/xml_format.cc +++ b/thin-provisioning/xml_format.cc @@ -50,6 +50,8 @@ namespace { void begin_superblock(string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) { @@ -57,6 +59,8 @@ namespace { out_ << "begin_superblock(get_attr(attr, "uuid"), get_attr(attr, "time"), get_attr(attr, "transaction"), + get_opt_attr(attr, "flags"), + get_opt_attr(attr, "version"), get_attr(attr, "data_block_size"), get_attr(attr, "nr_data_blocks"), get_opt_attr(attr, "metadata_snap")); From ced9929ca6d85b32b83d81fbc3148b94a5a10e87 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:31 +0800 Subject: [PATCH 044/118] [metadata_dumper] Support dump a bottom-level data mapping tree --- thin-provisioning/metadata_dumper.cc | 11 +++++++++++ thin-provisioning/metadata_dumper.h | 1 + 2 files changed, 12 insertions(+) diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index 5006fc0..9ba0614 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -244,3 +244,14 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) } //---------------------------------------------------------------- + +void +thin_provisioning::metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root) { + mapping_emitter me(e); + single_mapping_tree tree(*md->tm_, subtree_root, + mapping_tree_detail::block_time_ref_counter(md->data_sm_)); + walk_mapping_tree(tree, static_cast(me), + *mapping_damage_policy(repair)); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/metadata_dumper.h b/thin-provisioning/metadata_dumper.h index c96d22e..007256a 100644 --- a/thin-provisioning/metadata_dumper.h +++ b/thin-provisioning/metadata_dumper.h @@ -29,6 +29,7 @@ namespace thin_provisioning { // the dumper to do it's best to recover info. If not set, any // corruption encountered will cause an exception to be thrown. void metadata_dump(metadata::ptr md, emitter::ptr e, bool repair); + void metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root); } //---------------------------------------------------------------- From d28e64aff027606254b8d546218a17cbc3315fa1 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:45 +0800 Subject: [PATCH 045/118] [xml_parser] Allow element handlers to stop parsing --- base/xml_utils.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/xml_utils.cc b/base/xml_utils.cc index fb34153..edbd16a 100644 --- a/base/xml_utils.cc +++ b/base/xml_utils.cc @@ -19,13 +19,16 @@ xml_parser::parse(std::string const &backup_file, bool quiet) size_t total = 0; size_t input_length = get_file_length(backup_file); - while (!in.eof()) { + XML_Error error_code = XML_ERROR_NONE; + while (!in.eof() && error_code == XML_ERROR_NONE) { char buffer[4096]; in.read(buffer, sizeof(buffer)); size_t len = in.gcount(); int done = in.eof(); - if (!XML_Parse(parser_, buffer, len, done)) { + // Do not throw while normally aborted by element handlers + if (!XML_Parse(parser_, buffer, len, done) && + (error_code = XML_GetErrorCode(parser_)) != XML_ERROR_ABORTED) { ostringstream out; out << "Parse error at line " << XML_GetCurrentLineNumber(parser_) From 992ad02ce93594b5bfcaf18579f4dcf88b54ce2a Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:56 +0800 Subject: [PATCH 046/118] [restore_emitter] Recount device_details::mapped_blocks_ --- thin-provisioning/restore_emitter.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/thin-provisioning/restore_emitter.cc b/thin-provisioning/restore_emitter.cc index 2fab704..f6ca294 100644 --- a/thin-provisioning/restore_emitter.cc +++ b/thin-provisioning/restore_emitter.cc @@ -82,10 +82,11 @@ namespace { if (device_exists(dev)) throw std::runtime_error("Device already exists"); - // Add entry to the details tree - uint64_t key[1] = {dev}; - device_tree_detail::device_details details = {mapped_blocks, trans_id, (uint32_t)creation_time, (uint32_t)snap_time}; - md_->details_->insert(key, details); + // Store the entry of the details tree + current_device_details_.mapped_blocks_ = 0; + current_device_details_.transaction_id_ = trans_id; + current_device_details_.creation_time_ = (uint32_t)creation_time; + current_device_details_.snapshotted_time_ = (uint32_t)snap_time; current_mapping_ = empty_mapping_->clone(); current_device_ = boost::optional(dev); @@ -94,6 +95,9 @@ namespace { virtual void end_device() { uint64_t key[1] = {*current_device_}; + // Add entry to the details tree + md_->details_->insert(key, current_device_details_); + md_->mappings_top_level_->insert(key, current_mapping_->get_root()); md_->mappings_->set_root(md_->mappings_top_level_->get_root()); // FIXME: ugly @@ -134,6 +138,8 @@ namespace { bt.time_ = time; current_mapping_->insert(key, bt); md_->data_sm_->inc(data_block); + + current_device_details_.mapped_blocks_ += 1; } private: @@ -153,6 +159,7 @@ namespace { bool in_superblock_; block_address nr_data_blocks_; boost::optional current_device_; + device_tree_detail::device_details current_device_details_; single_mapping_tree::ptr current_mapping_; single_mapping_tree::ptr empty_mapping_; }; From 45e9916428f3f13c7ad704d3ea8a46154600f851 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:05 +0800 Subject: [PATCH 047/118] Expose validator creation interface --- persistent-data/space-maps/disk.cc | 15 ++++++++++----- persistent-data/space-maps/disk.h | 6 ++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/persistent-data/space-maps/disk.cc b/persistent-data/space-maps/disk.cc index e915eb4..cab9583 100644 --- a/persistent-data/space-maps/disk.cc +++ b/persistent-data/space-maps/disk.cc @@ -87,11 +87,6 @@ namespace { } }; - bcache::validator::ptr - index_validator() { - return bcache::validator::ptr(new index_block_validator()); - } - //-------------------------------- class bitmap { @@ -773,4 +768,14 @@ persistent_data::open_metadata_sm(transaction_manager &tm, void *root) checked_space_map::ptr(new sm_disk(store, tm, v)))); } +bcache::validator::ptr +persistent_data::bitmap_validator() { + return bcache::validator::ptr(new bitmap_block_validator()); +} + +bcache::validator::ptr +persistent_data::index_validator() { + return bcache::validator::ptr(new index_block_validator()); +} + //---------------------------------------------------------------- diff --git a/persistent-data/space-maps/disk.h b/persistent-data/space-maps/disk.h index 0a69f04..775ef7b 100644 --- a/persistent-data/space-maps/disk.h +++ b/persistent-data/space-maps/disk.h @@ -36,6 +36,12 @@ namespace persistent_data { checked_space_map::ptr open_metadata_sm(transaction_manager &tm, void *root); + + bcache::validator::ptr + bitmap_validator(); + + bcache::validator::ptr + index_validator(); } //---------------------------------------------------------------- From 778c153c1e71cd779b856e705492ae59fb9955e2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:13 +0800 Subject: [PATCH 048/118] [block-cache] Add check_raw() to bcache::validator --- block-cache/block_cache.h | 2 ++ caching/superblock.cc | 9 +++++++++ era/superblock.cc | 9 +++++++++ persistent-data/data-structures/array.h | 9 +++++++++ persistent-data/data-structures/btree.tcc | 1 - persistent-data/space-maps/disk.cc | 18 ++++++++++++++++++ persistent-data/validators.cc | 10 ++++++++++ thin-provisioning/superblock.cc | 9 +++++++++ unit-tests/block_t.cc | 8 ++++++++ 9 files changed, 74 insertions(+), 1 deletion(-) diff --git a/block-cache/block_cache.h b/block-cache/block_cache.h index 4bc6667..5de2c44 100644 --- a/block-cache/block_cache.h +++ b/block-cache/block_cache.h @@ -26,12 +26,14 @@ namespace bcache { virtual ~validator() {} virtual void check(void const *data, block_address location) const = 0; + virtual bool check_raw(void const *data) const = 0; virtual void prepare(void *data, block_address location) const = 0; }; class noop_validator : public validator { public: void check(void const *data, block_address location) const {} + bool check_raw(void const *data) const {return true;} void prepare(void *data, block_address location) const {} }; diff --git a/caching/superblock.cc b/caching/superblock.cc index 2edd6c1..3270b3f 100644 --- a/caching/superblock.cc +++ b/caching/superblock.cc @@ -292,6 +292,15 @@ namespace validator { throw checksum_error("bad checksum in superblock"); } + virtual bool check_raw(void const *raw) const { + superblock_disk const *sbd = reinterpret_cast(raw); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { superblock_disk *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); diff --git a/era/superblock.cc b/era/superblock.cc index e013064..f61a542 100644 --- a/era/superblock.cc +++ b/era/superblock.cc @@ -219,6 +219,15 @@ namespace era_validator { throw checksum_error("bad checksum in superblock"); } + virtual bool check_raw(void const *raw) const { + superblock_disk const *sbd = reinterpret_cast(raw); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { superblock_disk *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); diff --git a/persistent-data/data-structures/array.h b/persistent-data/data-structures/array.h index 1b87160..d7b1f70 100644 --- a/persistent-data/data-structures/array.h +++ b/persistent-data/data-structures/array.h @@ -43,6 +43,15 @@ namespace persistent_data { throw checksum_error("bad block nr in array block"); } + virtual bool check_raw(void const *raw) const { + array_block_disk const *data = reinterpret_cast(raw); + crc32c sum(ARRAY_CSUM_XOR); + sum.append(&data->max_entries, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(data->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { array_block_disk *data = reinterpret_cast(raw); data->blocknr = to_disk(location); diff --git a/persistent-data/data-structures/btree.tcc b/persistent-data/data-structures/btree.tcc index 06d0deb..3fc96e4 100644 --- a/persistent-data/data-structures/btree.tcc +++ b/persistent-data/data-structures/btree.tcc @@ -32,7 +32,6 @@ namespace { using namespace persistent_data; using namespace btree_detail; using namespace std; - } //---------------------------------------------------------------- diff --git a/persistent-data/space-maps/disk.cc b/persistent-data/space-maps/disk.cc index cab9583..ed7bae8 100644 --- a/persistent-data/space-maps/disk.cc +++ b/persistent-data/space-maps/disk.cc @@ -50,6 +50,15 @@ namespace { throw checksum_error("bad block nr in space map bitmap"); } + virtual bool check_raw(void const *raw) const { + bitmap_header const *data = reinterpret_cast(raw); + crc32c sum(BITMAP_CSUM_XOR); + sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(data->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { bitmap_header *data = reinterpret_cast(raw); data->blocknr = to_disk(location); @@ -77,6 +86,15 @@ namespace { throw checksum_error("bad block nr in metadata index block"); } + virtual bool check_raw(void const *raw) const { + metadata_index const *mi = reinterpret_cast(raw); + crc32c sum(INDEX_CSUM_XOR); + sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(mi->csum_)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { metadata_index *mi = reinterpret_cast(raw); mi->blocknr_ = to_disk(location); diff --git a/persistent-data/validators.cc b/persistent-data/validators.cc index b9c163c..a50947d 100644 --- a/persistent-data/validators.cc +++ b/persistent-data/validators.cc @@ -31,6 +31,16 @@ namespace { } } + virtual bool check_raw(void const *raw) const { + disk_node const *data = reinterpret_cast(raw); + node_header const *n = &data->header; + crc32c sum(BTREE_CSUM_XOR); + sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(n->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { disk_node *data = reinterpret_cast(raw); node_header *n = &data->header; diff --git a/thin-provisioning/superblock.cc b/thin-provisioning/superblock.cc index fb0d17e..f63ed52 100644 --- a/thin-provisioning/superblock.cc +++ b/thin-provisioning/superblock.cc @@ -98,6 +98,15 @@ namespace { } } + virtual bool check_raw(void const *raw) const { + superblock_disk const *sbd = reinterpret_cast(raw); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum_)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { superblock_disk *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); diff --git a/unit-tests/block_t.cc b/unit-tests/block_t.cc index c2a6d58..1a61d2b 100644 --- a/unit-tests/block_t.cc +++ b/unit-tests/block_t.cc @@ -44,6 +44,14 @@ namespace { throw runtime_error("validator check zero"); } + virtual bool check_raw(void const *raw) const { + unsigned char const *data = reinterpret_cast(raw); + for (unsigned b = 0; b < BlockSize; b++) + if (data[b] != 0) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { unsigned char *data = reinterpret_cast(raw); for (unsigned b = 0; b < BlockSize; b++) From a48227188e950c2f32ff5a5b64fa245844a2b969 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:23 +0800 Subject: [PATCH 049/118] Add utility class btree_detail::noop_damage_visitor --- persistent-data/data-structures/btree_damage_visitor.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index f0368bb..5a96d5d 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -27,6 +27,12 @@ namespace persistent_data { return out; } + class noop_damage_visitor { + public: + virtual void visit(btree_path const &path, damage const &d) { + } + }; + // Tracks damage in a single level btree. Use multiple // trackers if you have a multilayer tree. class damage_tracker { From c571a08f6ce90e14da82d155a85a240293f18f97 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:32 +0800 Subject: [PATCH 050/118] Add utility class btree_detail::noop_value_visitor --- .../data-structures/btree_base_visitor.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 persistent-data/data-structures/btree_base_visitor.h diff --git a/persistent-data/data-structures/btree_base_visitor.h b/persistent-data/data-structures/btree_base_visitor.h new file mode 100644 index 0000000..17519e1 --- /dev/null +++ b/persistent-data/data-structures/btree_base_visitor.h @@ -0,0 +1,17 @@ +#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H +#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H + +#include "persistent-data/data-structures/btree.h" + +namespace persistent_data { + namespace btree_detail { + template + class noop_value_visitor { + public: + virtual void visit(btree_path const &path, ValueType const &v) { + } + }; + } +} + +#endif From ad03114bf7703bb47bac508e4cab5a62792bb77a Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:44 +0800 Subject: [PATCH 051/118] Add utility class btree_detail::key_value_extractor --- .../btree_key_value_extractor.h | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 persistent-data/data-structures/btree_key_value_extractor.h diff --git a/persistent-data/data-structures/btree_key_value_extractor.h b/persistent-data/data-structures/btree_key_value_extractor.h new file mode 100644 index 0000000..40b7f7b --- /dev/null +++ b/persistent-data/data-structures/btree_key_value_extractor.h @@ -0,0 +1,39 @@ +#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H +#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H + +#include "persistent-data/data-structures/btree_damage_visitor.h" +#include + +namespace persistent_data { + namespace btree_detail { + template + class key_value_extractor { + typedef typename std::map MapType; + public: + key_value_extractor(MapType &map): map_(map) { + } + + virtual ~key_value_extractor() { + } + + virtual void visit(btree_path const &path, ValueType const &v) { + map_.insert(std::make_pair(path.back(), v)); + } + private: + MapType &map_; + }; + + template + void btree_extract_key_values(btree const &tree, + std::map &map) { + typedef key_value_extractor KeyValueExtractor; + KeyValueExtractor kve(map); + noop_damage_visitor noop_dv; + btree_detail::btree_damage_visitor + v(kve, noop_dv); + tree.visit_depth_first(v); + } + } +} + +#endif From b22495997a8bf427741225ef86edf267ee746d7c Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:22:00 +0800 Subject: [PATCH 052/118] Allow counting_visitor to work with damaged btrees --- .../data-structures/btree_counter.h | 28 +++++++++++++------ .../data-structures/btree_damage_visitor.h | 5 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index ed7b845..c215626 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -2,35 +2,44 @@ #define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H #include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_base_visitor.h" +#include "persistent-data/data-structures/btree_damage_visitor.h" #include "persistent-data/block_counter.h" //---------------------------------------------------------------- namespace persistent_data { namespace btree_count_detail { - template - class counting_visitor : public btree::visitor { + template + class counting_visitor : public btree_damage_visitor { + typedef btree_damage_visitor BtreeDamageVisitor; public: typedef btree tree; - counting_visitor(block_counter &bc, ValueCounter &vc) - : bc_(bc), + counting_visitor(ValueVisitor &value_visitor, + DamageVisitor &damage_visitor, + block_counter &bc, + ValueCounter &vc) + : BtreeDamageVisitor(value_visitor, damage_visitor, false), + bc_(bc), vc_(vc) { } virtual bool visit_internal(node_location const &l, typename tree::internal_node const &n) { - return visit_node(n); + return BtreeDamageVisitor::visit_internal(l, n) ? + visit_node(n) : false; } virtual bool visit_internal_leaf(node_location const &l, typename tree::internal_node const &n) { - return visit_node(n); + return BtreeDamageVisitor::visit_internal_leaf(l, n) ? + visit_node(n) : false; } virtual bool visit_leaf(node_location const &l, typename tree::leaf_node const &n) { - if (visit_node(n)) { + if (BtreeDamageVisitor::visit_leaf(l, n) && visit_node(n)) { unsigned nr = n.get_nr_entries(); for (unsigned i = 0; i < nr; i++) { @@ -85,7 +94,10 @@ namespace persistent_data { // is not corrupt. template void count_btree_blocks(btree const &tree, block_counter &bc, ValueCounter &vc) { - btree_count_detail::counting_visitor v(bc, vc); + typedef noop_value_visitor NoopValueVisitor; + NoopValueVisitor noop_vv; + noop_damage_visitor noop_dv; + btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); tree.visit_depth_first(v); } } diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 5a96d5d..8be623b 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -158,8 +158,9 @@ namespace persistent_data { typedef boost::optional maybe_run64; btree_damage_visitor(ValueVisitor &value_visitor, - DamageVisitor &damage_visitor) - : avoid_repeated_visits_(true), + DamageVisitor &damage_visitor, + bool avoid_repeated_visits = true) + : avoid_repeated_visits_(avoid_repeated_visits), value_visitor_(value_visitor), damage_visitor_(damage_visitor) { } From d068ec8082928001e4abfbbe5490236df9485a87 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:22:09 +0800 Subject: [PATCH 053/118] Add utility class binary_block_counter --- persistent-data/block_counter.h | 30 +++++++++++++++++-- .../data-structures/btree_counter.h | 11 +++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/persistent-data/block_counter.h b/persistent-data/block_counter.h index e4232b4..ea70f93 100644 --- a/persistent-data/block_counter.h +++ b/persistent-data/block_counter.h @@ -20,6 +20,7 @@ #define BLOCK_COUNTER_H #include "block.h" +#include "run_set.h" //---------------------------------------------------------------- @@ -32,7 +33,9 @@ namespace persistent_data { public: typedef std::map count_map; - void inc(block_address b) { + virtual ~block_counter() {} + + virtual void inc(block_address b) { count_map::iterator it = counts_.find(b); if (it == counts_.end()) counts_.insert(make_pair(b, 1)); @@ -40,7 +43,7 @@ namespace persistent_data { it->second++; } - unsigned get_count(block_address b) const { + virtual unsigned get_count(block_address b) const { count_map::const_iterator it = counts_.find(b); return (it == counts_.end()) ? 0 : it->second; } @@ -52,6 +55,29 @@ namespace persistent_data { private: count_map counts_; }; + + //---------------------------------------------------------------- + // Little helper class that keeps track of which blocks + // are referenced. + //---------------------------------------------------------------- + class binary_block_counter : public block_counter { + public: + virtual ~binary_block_counter() {} + + virtual void inc(block_address b) { + visited_.add(b); + } + + virtual unsigned get_count(block_address b) const { + return visited_.member(b) ? 1 : 0; + } + + base::run_set const& get_visited() const { + return visited_; + } + private: + base::run_set visited_; + }; } //---------------------------------------------------------------- diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index c215626..6e52c29 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -100,6 +100,17 @@ namespace persistent_data { btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); tree.visit_depth_first(v); } + + template + void count_btree_blocks(btree const &tree, block_counter &bc) { + typedef noop_value_visitor NoopValueVisitor; + NoopValueVisitor noop_vv; + noop_damage_visitor noop_dv; + typedef noop_value_counter NoopValueCounter; + NoopValueCounter vc; + btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); + tree.visit_depth_first(v); + } } //---------------------------------------------------------------- From 80783e77292c4540cb005b855a735fe05c82bbbe Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:23:06 +0800 Subject: [PATCH 054/118] Add metadata_counter.{h,cc} --- Makefile.in | 1 + thin-provisioning/metadata_counter.cc | 73 +++++++++++++++++++++++++++ thin-provisioning/metadata_counter.h | 24 +++++++++ 3 files changed, 98 insertions(+) create mode 100644 thin-provisioning/metadata_counter.cc create mode 100644 thin-provisioning/metadata_counter.h diff --git a/Makefile.in b/Makefile.in index 30b0c1a..b515463 100644 --- a/Makefile.in +++ b/Makefile.in @@ -81,6 +81,7 @@ SOURCE=\ thin-provisioning/mapping_tree.cc \ thin-provisioning/metadata.cc \ thin-provisioning/metadata_checker.cc \ + thin-provisioning/metadata_counter.cc \ thin-provisioning/metadata_dumper.cc \ thin-provisioning/restore_emitter.cc \ thin-provisioning/rmap_visitor.cc \ diff --git a/thin-provisioning/metadata_counter.cc b/thin-provisioning/metadata_counter.cc new file mode 100644 index 0000000..bf3f809 --- /dev/null +++ b/thin-provisioning/metadata_counter.cc @@ -0,0 +1,73 @@ +#include "thin-provisioning/metadata_counter.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk_structures.h" + +using namespace persistent_data; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +void thin_provisioning::count_trees(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc) { + + // Count the device tree + { + noop_value_counter vc; + device_tree dtree(*tm, sb.device_details_root_, + device_tree_detail::device_details_traits::ref_counter()); + count_btree_blocks(dtree, bc, vc); + } + + // Count the mapping tree + { + noop_value_counter vc; + mapping_tree mtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::block_traits::ref_counter(space_map::ptr())); + count_btree_blocks(mtree, bc, vc); + } +} + +void thin_provisioning::count_space_maps(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc) { + // Count the metadata space map (no-throw) + try { + persistent_space_map::ptr metadata_sm = + open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); + metadata_sm->count_metadata(bc); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + // Count the data space map (no-throw) + try { + persistent_space_map::ptr data_sm = + open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); + data_sm->count_metadata(bc); + } catch (std::exception &e) { + cerr << e.what() << endl; + } +} + +void thin_provisioning::count_metadata(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc, + bool skip_metadata_snap) { + // Count the superblock + bc.inc(superblock_detail::SUPERBLOCK_LOCATION); + count_trees(tm, sb, bc); + + // Count the metadata snap, if present + if (!skip_metadata_snap && sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) { + bc.inc(sb.metadata_snap_); + + superblock_detail::superblock snap = read_superblock(tm->get_bm(), sb.metadata_snap_); + count_trees(tm, snap, bc); + } + + count_trees(tm, sb, bc); + count_space_maps(tm, sb, bc); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/metadata_counter.h b/thin-provisioning/metadata_counter.h new file mode 100644 index 0000000..861d72f --- /dev/null +++ b/thin-provisioning/metadata_counter.h @@ -0,0 +1,24 @@ +#ifndef METADATA_COUNTER_H +#define METADATA_COUNTER_H + +#include "thin-provisioning/metadata.h" +#include "persistent-data/data-structures/btree_counter.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + void count_trees(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc); + void count_space_maps(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc); + void count_metadata(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc, + bool skip_metadata_snap = false); +} + +//---------------------------------------------------------------- + +#endif From b05b9aa2275a894c5e716a22413f705f6943b0bf Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:24:14 +0800 Subject: [PATCH 055/118] [thin_scan] first draft --- Makefile.in | 1 + thin-provisioning/commands.cc | 1 + thin-provisioning/commands.h | 7 + thin-provisioning/thin_scan.cc | 417 +++++++++++++++++++++++++++++++++ 4 files changed, 426 insertions(+) create mode 100644 thin-provisioning/thin_scan.cc diff --git a/Makefile.in b/Makefile.in index b515463..931cb38 100644 --- a/Makefile.in +++ b/Makefile.in @@ -95,6 +95,7 @@ SOURCE=\ thin-provisioning/thin_repair.cc \ thin-provisioning/thin_restore.cc \ thin-provisioning/thin_rmap.cc \ + thin-provisioning/thin_scan.cc \ thin-provisioning/thin_trim.cc \ thin-provisioning/xml_format.cc diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index c25abce..6d2f7e4 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -16,6 +16,7 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_restore_cmd())); app.add_cmd(command::ptr(new thin_repair_cmd())); app.add_cmd(command::ptr(new thin_rmap_cmd())); + app.add_cmd(command::ptr(new thin_scan_cmd())); app.add_cmd(command::ptr(new thin_trim_cmd())); } diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index ec1f1ec..a8fa4a4 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -63,6 +63,13 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_scan_cmd : public base::command { + public: + thin_scan_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_trim_cmd : public base::command { public: thin_trim_cmd(); diff --git a/thin-provisioning/thin_scan.cc b/thin-provisioning/thin_scan.cc new file mode 100644 index 0000000..c5afbd9 --- /dev/null +++ b/thin-provisioning/thin_scan.cc @@ -0,0 +1,417 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include +#include + +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/superblock.h" +#include "thin-provisioning/commands.h" +#include "version.h" + +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + // extracted from btree_damage_visitor.h + template + bool check_block_nr(node const &n) { + if (n.get_location() != n.get_block_nr()) { + return false; + } + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_max_entries(node const &n) { + size_t elt_size = sizeof(uint64_t) + n.get_value_size(); + if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { + return false; + } + + if (n.get_max_entries() % 3) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_nr_entries(node const &n, bool is_root) { + if (n.get_nr_entries() > n.get_max_entries()) { + return false; + } + + block_address min = n.get_max_entries() / 3; + if (!is_root && (n.get_nr_entries() < min)) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_ordered_keys(node const &n) { + unsigned nr_entries = n.get_nr_entries(); + + if (nr_entries == 0) + return true; // can only happen if a root node + + uint64_t last_key = n.key_at(0); + + for (unsigned i = 1; i < nr_entries; i++) { + uint64_t k = n.key_at(i); + if (k <= last_key) { + return false; + } + last_key = k; + } + + return true; + } + + transaction_manager::ptr + open_tm(block_manager<>::ptr bm) { + space_map::ptr sm(new core_map(bm->get_nr_blocks())); + sm->inc(superblock_detail::SUPERBLOCK_LOCATION); + transaction_manager::ptr tm(new transaction_manager(bm, sm)); + return tm; + } +} + +namespace { + // FIXME: deprecated conversion from string constant to ‘char*’ + char const* metadata_block_type_name[] = { + "unknown", + "zero", + "superblock", + "btree_internal", + "btree_leaf", + "btree_unknown", + "index_block", + "bitmap_block" + }; + + enum metadata_block_type { + UNKNOWN = 0, + ZERO, + SUPERBLOCK, + BTREE_INTERNAL, + BTREE_LEAF, + BTREE_UNKNOWN, + INDEX_BLOCK, + BITMAP_BLOCK + }; + + struct block_range { + uint64_t begin_; + uint64_t end_; // one-pass-the-end + boost::optional blocknr_begin_; + metadata_block_type type_; + int64_t ref_count_; // ref_count in metadata space map + size_t value_size_; // btree node only + bool is_valid_; // btree node only + + block_range() + : begin_(0), end_(0), + type_(UNKNOWN), ref_count_(-1), + value_size_(0), is_valid_(false) + { + } + + block_range(block_range const &rhs) + : begin_(rhs.begin_), end_(rhs.end_), + blocknr_begin_(rhs.blocknr_begin_), + type_(rhs.type_), ref_count_(rhs.ref_count_), + value_size_(rhs.value_size_), is_valid_(rhs.is_valid_) + { + } + }; + + void output_block_range(block_range const &r, std::ostream &out) { + if (r.end_ <= r.begin_) + return; + + if (r.end_ - r.begin_ > 1) { + out << "" << endl; + } else + out << "\"/>" << endl; + } + + //------------------------------------------------------------------- + + struct flags { + flags() { + } + + boost::optional scan_begin_; + boost::optional scan_end_; + }; + + int scan_metadata_(string const &input, + std::ostream &out, + flags const &f) { + using namespace persistent_data; + using namespace thin_provisioning; + using namespace sm_disk_detail; + + block_manager<>::ptr bm; + bm = open_bm(input, block_manager<>::READ_ONLY); + + block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0; + block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks(); + + const std::vector zeros(MD_BLOCK_SIZE, 0); + + // try to open metadata space-map (it's okay to fail) + // note: transaction_manager and space_map must be in the same scope + transaction_manager::ptr tm; + checked_space_map::ptr metadata_sm; + try { + superblock_detail::superblock sb = read_superblock(bm); + tm = open_tm(bm); + metadata_sm = open_metadata_sm(*tm, &sb.metadata_space_map_root_); + tm->set_sm(metadata_sm); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + block_range curr_range; + block_range run_range; + + bcache::validator::ptr sv = superblock_validator(); + bcache::validator::ptr nv = create_btree_node_validator(); + bcache::validator::ptr iv = index_validator(); + bcache::validator::ptr bv = bitmap_validator(); + + for (block_address b = scan_begin; b < scan_end; ++b) { + block_manager<>::read_ref rr = bm->read_lock(b); + + curr_range.begin_ = b; + curr_range.end_ = b + 1; + curr_range.blocknr_begin_ = boost::none; + curr_range.type_ = UNKNOWN; + curr_range.is_valid_ = false; + + if (!memcmp(rr.data(), zeros.data(), MD_BLOCK_SIZE)) + curr_range.type_ = ZERO; + + if (curr_range.type_ == UNKNOWN && sv->check_raw(rr.data())) { + curr_range.type_ = SUPERBLOCK; + curr_range.is_valid_ = true; + } + + if (curr_range.type_ == UNKNOWN && nv->check_raw(rr.data())) { + // note: check_raw() doesn't check node_header::blocknr_ + node_ref n = btree_detail::to_node(rr); + uint32_t flags = to_cpu(n.raw()->header.flags); + if ((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) + curr_range.type_ = BTREE_INTERNAL; + else if (flags & LEAF_NODE) + curr_range.type_ = BTREE_LEAF; + else + curr_range.type_ = BTREE_UNKNOWN; + + if (curr_range.type_ != BTREE_UNKNOWN && + check_block_nr(n) && + check_max_entries(n) && + check_nr_entries(n, true) && + check_ordered_keys(n)) + curr_range.is_valid_ = true; + else + curr_range.is_valid_ = false; + + curr_range.blocknr_begin_ = n.get_block_nr(); + curr_range.value_size_ = n.get_value_size(); + } + + if (curr_range.type_ == UNKNOWN && bv->check_raw(rr.data())) { + curr_range.type_ = BITMAP_BLOCK; + bitmap_header const *data = reinterpret_cast(rr.data()); + curr_range.blocknr_begin_ = to_cpu(data->blocknr); + curr_range.is_valid_ = (to_cpu(data->blocknr) == b) ? true : false; + } + + if (curr_range.type_ == UNKNOWN && iv->check_raw(rr.data())) { + curr_range.type_ = INDEX_BLOCK; + metadata_index const *mi = reinterpret_cast(rr.data()); + curr_range.blocknr_begin_ = to_cpu(mi->blocknr_); + curr_range.is_valid_ = (to_cpu(mi->blocknr_) == b) ? true : false; + } + + try { + curr_range.ref_count_ = metadata_sm ? + static_cast(metadata_sm->get_count(b)) : -1; + } catch (std::exception &e) { + curr_range.ref_count_ = -1; + } + + // output the current block + if (run_range.end_ == 0) + run_range = curr_range; + else if (((!curr_range.blocknr_begin_ && !run_range.blocknr_begin_) || + (curr_range.blocknr_begin_ && run_range.blocknr_begin_ && + *curr_range.blocknr_begin_ == *run_range.blocknr_begin_ + (run_range.end_ - run_range.begin_))) && + curr_range.type_ == run_range.type_ && + curr_range.ref_count_ == run_range.ref_count_ && + curr_range.value_size_ == run_range.value_size_ && + curr_range.is_valid_ == run_range.is_valid_) { + ++run_range.end_; + } else { + output_block_range(run_range, out); + run_range = curr_range; + } + } + + // output the last run + output_block_range(run_range, out); + + return 0; + } + + int scan_metadata(string const &input, + boost::optional output, + flags const &f) { + try { + if (output) { + std::ofstream out(output->c_str()); + scan_metadata_(input, out, f); + } else + scan_metadata_(input, cout, f); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_scan_cmd::thin_scan_cmd() + : command("thin_scan") +{ +} + +void +thin_scan_cmd::usage(std::ostream &out) const { + out << "Usage: " << get_name() << " [options] {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-o|--output} " << endl + << " {--begin} " << endl + << " {--end} " << endl + << " {-V|--version}" << endl; +} + +int +thin_scan_cmd::run(int argc, char **argv) +{ + const char shortopts[] = "ho:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "output", required_argument, NULL, 'o'}, + { "version", no_argument, NULL, 'V'}, + { "begin", required_argument, NULL, 1}, + { "end", required_argument, NULL, 2}, + { NULL, no_argument, NULL, 0 } + }; + boost::optional output; + flags f; + + char c; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'o': + output = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + case 1: + try { + f.scan_begin_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 2: + try { + f.scan_end_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + default: + usage(cerr); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr); + return 1; + } + + if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) { + cerr << "badly formed region (end <= begin)" << endl; + return 1; + } + + return scan_metadata(argv[optind], output, f); +} + +//--------------------------------------------------------------------------- From 08a7093cd51b76e792b23b07d69f2cedb92d214b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:24:28 +0800 Subject: [PATCH 056/118] [thin_ll_dump][thin_ll_restore] first draft --- Makefile.in | 2 + thin-provisioning/commands.cc | 2 + thin-provisioning/commands.h | 14 + thin-provisioning/thin_ll_dump.cc | 473 +++++++++++++++++++++++++++ thin-provisioning/thin_ll_restore.cc | 276 ++++++++++++++++ 5 files changed, 767 insertions(+) create mode 100644 thin-provisioning/thin_ll_dump.cc create mode 100644 thin-provisioning/thin_ll_restore.cc diff --git a/Makefile.in b/Makefile.in index 931cb38..542154d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -89,6 +89,8 @@ SOURCE=\ thin-provisioning/thin_check.cc \ thin-provisioning/thin_delta.cc \ thin-provisioning/thin_dump.cc \ + thin-provisioning/thin_ll_dump.cc \ + thin-provisioning/thin_ll_restore.cc \ thin-provisioning/thin_ls.cc \ thin-provisioning/thin_metadata_size.cc \ thin-provisioning/thin_pool.cc \ diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index 6d2f7e4..ba240c9 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -11,6 +11,8 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_check_cmd())); app.add_cmd(command::ptr(new thin_delta_cmd())); app.add_cmd(command::ptr(new thin_dump_cmd())); + app.add_cmd(command::ptr(new thin_ll_dump_cmd())); + app.add_cmd(command::ptr(new thin_ll_restore_cmd())); app.add_cmd(command::ptr(new thin_ls_cmd())); app.add_cmd(command::ptr(new thin_metadata_size_cmd())); app.add_cmd(command::ptr(new thin_restore_cmd())); diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index a8fa4a4..d6ed78f 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -28,6 +28,20 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_ll_dump_cmd : public base::command { + public: + thin_ll_dump_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_ll_restore_cmd : public base::command { + public: + thin_ll_restore_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_ls_cmd : public base::command { public: thin_ls_cmd(); diff --git a/thin-provisioning/thin_ll_dump.cc b/thin-provisioning/thin_ll_dump.cc new file mode 100644 index 0000000..7f7b14c --- /dev/null +++ b/thin-provisioning/thin_ll_dump.cc @@ -0,0 +1,473 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include +#include + +#include "base/indented_stream.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_counter.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/metadata_counter.h" +#include "thin-provisioning/commands.h" +#include "version.h" + +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + // extracted from btree_damage_visitor.h + template + bool check_block_nr(node const &n) { + if (n.get_location() != n.get_block_nr()) { + return false; + } + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_max_entries(node const &n) { + size_t elt_size = sizeof(uint64_t) + n.get_value_size(); + if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { + return false; + } + + if (n.get_max_entries() % 3) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_nr_entries(node const &n, bool is_root) { + if (n.get_nr_entries() > n.get_max_entries()) { + return false; + } + + block_address min = n.get_max_entries() / 3; + if (!is_root && (n.get_nr_entries() < min)) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_ordered_keys(node const &n) { + unsigned nr_entries = n.get_nr_entries(); + + if (nr_entries == 0) + return true; // can only happen if a root node + + uint64_t last_key = n.key_at(0); + + for (unsigned i = 1; i < nr_entries; i++) { + uint64_t k = n.key_at(i); + if (k <= last_key) { + return false; + } + last_key = k; + } + + return true; + } + + transaction_manager::ptr + open_tm(block_manager<>::ptr bm) { + space_map::ptr sm(new core_map(bm->get_nr_blocks())); + sm->inc(superblock_detail::SUPERBLOCK_LOCATION); + transaction_manager::ptr tm(new transaction_manager(bm, sm)); + return tm; + } +} + +//--------------------------------------------------------------------------- + +namespace { + struct node_info { + uint64_t blocknr_; + uint32_t flags_; + uint64_t key_begin_; + uint64_t key_end_; + uint64_t nr_entries_; + uint32_t value_size_; + }; + + //------------------------------------------------------------------- + + struct btree_node_checker { + typedef boost::shared_ptr ptr; + virtual ~btree_node_checker() {} + virtual bool check(node_ref &n) = 0; + }; + + struct unvisited_btree_node_filter: public btree_node_checker { + unvisited_btree_node_filter(block_counter const &bc) + : nv_(create_btree_node_validator()), bc_(bc) { + } + + virtual bool check(node_ref &n) { + uint32_t flags = to_cpu(n.raw()->header.flags); + if ((n.get_value_size() == sizeof(mapping_tree_detail::block_traits::disk_type) || + n.get_value_size() == sizeof(device_tree_detail::device_details_traits::disk_type)) && + !bc_.get_count(n.get_location()) && + check_block_nr(n) && + (((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) || + (flags & LEAF_NODE)) && + nv_->check_raw(n.raw()) && + check_max_entries(n) && + check_nr_entries(n, true) && + check_ordered_keys(n)) + return true; + return false; + } + + bcache::validator::ptr nv_; + block_counter const &bc_; + }; + + //------------------------------------------------------------------- + + void find_btree_nodes(block_manager<>::ptr bm, + block_address begin, + block_address end, + btree_node_checker::ptr checker, + base::run_set &found) { + using namespace persistent_data; + + for (block_address b = begin; b < end; ++b) { + block_manager<>::read_ref rr = bm->read_lock(b); + node_ref n = btree_detail::to_node(rr); + + if (checker->check(n)) + found.add(b); + } + } + + //------------------------------------------------------------------- + + bool first_key_cmp(node_info const &lhs, node_info const &rhs) { + return lhs.key_begin_ < rhs.key_begin_; + } + + template + void convert_to_node_info(node_ref const &n, node_info &ni) { + ni.blocknr_ = n.get_location(); + ni.flags_ = to_cpu(n.raw()->header.flags); + if ((ni.nr_entries_ = n.get_nr_entries()) > 0) { + ni.key_begin_ = n.key_at(0); + ni.key_end_ = n.key_at(n.get_nr_entries() - 1); + } + ni.value_size_ = n.get_value_size(); + } + + void output_node_info(indented_stream &out, node_info const &ni) { + out.indent(); + out << "" << endl; + } + + //------------------------------------------------------------------- + + class ll_mapping_tree_emitter : public mapping_tree_detail::device_visitor { + public: + ll_mapping_tree_emitter(block_manager<>::ptr bm, + indented_stream &out) + : bm_(bm), out_(out) { + } + + void visit(btree_path const &path, block_address tree_root) { + out_.indent(); + out_ << "" << endl; + out_.inc(); + + // Do not throw exception. Process the next entry inside the current node. + try { + block_manager<>::read_ref rr = bm_->read_lock(tree_root); + node_ref n = btree_detail::to_node(rr); + node_info ni; + convert_to_node_info(n, ni); + output_node_info(out_, ni); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + out_.dec(); + out_.indent(); + out_ << "" << endl; + } + private: + block_manager<>::ptr bm_; + indented_stream& out_; + }; + + //------------------------------------------------------------------- + + struct flags { + flags() : use_metadata_snap_(false) { + } + + bool use_metadata_snap_; + boost::optional metadata_snap_; + boost::optional data_mapping_root_; + boost::optional device_details_root_; + boost::optional scan_begin_; + boost::optional scan_end_; + }; + + int low_level_dump_(string const &input, + std::ostream &output, + flags const &f) { + block_manager<>::ptr bm = open_bm(input, block_manager<>::READ_ONLY); + + block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0; + block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks(); + + // Allow to read superblock at arbitrary location for low-level dump, + // without checking equality between the given metadata_snap and sb.metadata_snap_ + superblock_detail::superblock sb = read_superblock(bm, superblock_detail::SUPERBLOCK_LOCATION); + if (f.use_metadata_snap_) { + sb = f.metadata_snap_ ? + read_superblock(bm, *f.metadata_snap_) : + read_superblock(bm, sb.metadata_snap_); + } + // override sb.data_mapping_root_ + if (f.data_mapping_root_) + sb.data_mapping_root_ = *f.data_mapping_root_; + // override sb.device_details_root_ + if (f.device_details_root_) + sb.device_details_root_ = *f.device_details_root_; + + transaction_manager::ptr tm = open_tm(bm); + + indented_stream out(output); + + out.indent(); + out << "" << endl; + out.inc(); + + // output the top-level data mapping tree + ll_mapping_tree_emitter ll_mte(tm->get_bm(), out); + dev_tree dtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::mtree_traits::ref_counter(tm)); + noop_damage_visitor noop_dv; + btree_visit_values(dtree, ll_mte, noop_dv); + + out.dec(); + out.indent(); + out << "" << endl; + + // find orphans + binary_block_counter bc; + bc.inc(superblock_detail::SUPERBLOCK_LOCATION); + count_metadata(tm, sb, bc, true); + btree_node_checker::ptr filter = btree_node_checker::ptr( + new unvisited_btree_node_filter(bc)); + base::run_set orphans; + find_btree_nodes(bm, scan_begin, scan_end, filter, orphans); + + // sort orphans + std::vector nodes; + for (base::run_set::const_iterator it = orphans.begin(); + it != orphans.end(); + ++it) { + if (it->begin_ && it->end_) { + for (block_address b = *it->begin_; b < *it->end_; ++b) { + block_manager<>::read_ref rr = bm->read_lock(b); + node_ref n = btree_detail::to_node(rr); + nodes.push_back(node_info()); + convert_to_node_info(n, nodes.back()); + } + } + } + std::sort(nodes.begin(), nodes.end(), first_key_cmp); + + // output orphans + out.indent(); + out << "" << std::endl; + out.inc(); + + for (size_t i = 0; i < nodes.size(); ++i) + output_node_info(out, nodes[i]); + + out.dec(); + out.indent(); + out << "" << std::endl; + + return 0; + } + + int low_level_dump(string const &input, + boost::optional output, + flags const &f) { + try { + if (output) { + ofstream out(output->c_str()); + low_level_dump_(input, out, f); + } else + low_level_dump_(input, cout, f); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_ll_dump_cmd::thin_ll_dump_cmd() + : command("thin_ll_dump") +{ +} + +void +thin_ll_dump_cmd::usage(ostream &out) const { + out << "Usage: " << get_name() << " [options] {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-m|--metadata-snap}[block#]" << endl + << " {-o|--output} " << endl + << " {--begin} " << endl + << " {--end} " << endl + << " {--data-mapping-root} " << endl + << " {--device-details-root} " << endl + << " {-V|--version}" << endl; +} + +int +thin_ll_dump_cmd::run(int argc, char **argv) +{ + const char shortopts[] = "hm:o:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "metadata-snap", optional_argument, NULL, 'm'}, + { "output", required_argument, NULL, 'o'}, + { "version", no_argument, NULL, 'V'}, + { "begin", required_argument, NULL, 1}, + { "end", required_argument, NULL, 2}, + { "data-mapping-root", required_argument, NULL, 3}, + { "device-details-root", required_argument, NULL, 4}, + { NULL, no_argument, NULL, 0 } + }; + boost::optional output; + flags f; + + char c; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'm': + f.use_metadata_snap_ = true; + if (optarg) { + try { + f.metadata_snap_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + } + break; + + case 'o': + output = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + case 1: + try { + f.scan_begin_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 2: + try { + f.scan_end_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 3: + try { + f.data_mapping_root_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 4: + try { + f.device_details_root_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + default: + usage(cerr); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr); + return 1; + } + + if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) { + cerr << "badly formed region (end <= begin)" << endl; + usage(cerr); + return 1; + } + + return low_level_dump(argv[optind], output, f); +} + +//--------------------------------------------------------------------------- diff --git a/thin-provisioning/thin_ll_restore.cc b/thin-provisioning/thin_ll_restore.cc new file mode 100644 index 0000000..1168579 --- /dev/null +++ b/thin-provisioning/thin_ll_restore.cc @@ -0,0 +1,276 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include "base/xml_utils.h" +#include "metadata_dumper.h" +#include "metadata.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "restore_emitter.h" +#include "xml_format.h" +#include "thin-provisioning/commands.h" +#include "version.h" + +#include +#include +#include + +using namespace persistent_data; +using namespace std; +using namespace thin_provisioning; +using namespace xml_utils; + +//---------------------------------------------------------------- + +namespace { + struct user_data { + block_manager<>::ptr input_bm_; + block_manager<>::ptr output_bm_; + + metadata::ptr md_; + XML_Parser parser_; + emitter::ptr emitter_; + }; + + void open_resources(user_data &ud, attributes const &attr) { + boost::optional val; + + // open the input metadata + // Allow to read superblock at arbitrary location for low-level restore + block_address sb_location = (val = get_opt_attr(attr, "blocknr")) ? + *val : superblock_detail::SUPERBLOCK_LOCATION; + ud.md_ = metadata::ptr(new metadata(ud.input_bm_, sb_location)); + + // override superblock::device_details_root_ + if ((val = get_opt_attr(attr, "device_details_root"))) { + ud.md_->sb_.device_details_root_ = *val; + ud.md_->details_ = device_tree::ptr(new device_tree(*ud.md_->tm_, *val, + device_tree_detail::device_details_traits::ref_counter())); + } + + // open the output metadata + metadata::ptr new_md(new metadata(ud.output_bm_, metadata::CREATE, 128, 0)); + + ud.emitter_ = create_restore_emitter(new_md); + } + + void parse_superblock(metadata::ptr md, emitter::ptr e, attributes const &attr) { + sm_disk_detail::sm_root_disk const *d = + reinterpret_cast(md->sb_.data_space_map_root_); + sm_disk_detail::sm_root v; + sm_disk_detail::sm_root_traits::unpack(*d, v); + + e->begin_superblock("", md->sb_.time_, + md->sb_.trans_id_, + md->sb_.flags_, + md->sb_.version_, + md->sb_.data_block_size_, + v.nr_blocks_, + boost::optional()); + } + + void parse_device(metadata::ptr md, emitter::ptr e, attributes const &attr) { + uint32_t dev_id = get_attr(attr, "dev_id"); + device_tree_detail::device_details details; + details.transaction_id_ = 0; + details.creation_time_ = 0; + details.snapshotted_time_ = 0; + + device_tree::ptr details_tree; + boost::optional details_root = get_opt_attr(attr, "blocknr"); + if (details_root) + details_tree = device_tree::ptr(new device_tree(*md->tm_, *details_root, + device_tree_detail::device_details_traits::ref_counter())); + else + details_tree = md->details_; + + uint64_t key[1] = {dev_id}; + device_tree::maybe_value v; + try { + v = details_tree->lookup(key); + } catch (std::exception &e) { + cerr << "missing device " << dev_id << ": " << e.what() << endl; + } + if (v) + details = *v; + + e->begin_device(dev_id, + 0, + details.transaction_id_, + details.creation_time_, + details.snapshotted_time_); + } + + void parse_node(metadata::ptr md, emitter::ptr e, attributes const &attr) { + metadata_dump_subtree(md, e, true, get_attr(attr, "blocknr")); + } + + void start_tag(void *data, char const *el, char const **attr) { + user_data *ud = static_cast(data); + attributes a; + + build_attributes(a, attr); + + if (!strcmp(el, "superblock")) { + open_resources(*ud, a); + parse_superblock(ud->md_, ud->emitter_, a); + + } else if (!strcmp(el, "device")) + parse_device(ud->md_, ud->emitter_, a); + + else if (!strcmp(el, "node")) + parse_node(ud->md_, ud->emitter_, a); + + else + throw runtime_error("unknown tag type"); + } + + void end_tag(void *data, const char *el) { + user_data *ud = static_cast(data); + + if (!strcmp(el, "superblock")) { + ud->emitter_->end_superblock(); + XML_StopParser(ud->parser_, XML_FALSE); // skip the rest elements + } + + else if (!strcmp(el, "device")) + ud->emitter_->end_device(); + + else if (!strcmp(el, "node")) + ; + + else + throw runtime_error("unknown tag type"); + } +} + +//--------------------------------------------------------------------------- + +namespace { + struct flags { + flags() { + } + }; + + int low_level_restore_(string const &src_metadata, string const &input, + string const &output, flags const &f) { + user_data ud; + ud.input_bm_ = open_bm(src_metadata, block_manager<>::READ_ONLY); + ud.output_bm_ = open_bm(output, block_manager<>::READ_WRITE); + + xml_parser p; + ud.parser_ = p.get_parser(); + + XML_SetUserData(p.get_parser(), &ud); + XML_SetElementHandler(p.get_parser(), start_tag, end_tag); + + bool quiet = true; + p.parse(input, quiet); + + return 0; + } + + int low_level_restore(string const &src_metadata, string const &input, + string const &output, flags const &f) { + try { + low_level_restore_(src_metadata, input, output, f); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_ll_restore_cmd::thin_ll_restore_cmd() + : command("thin_ll_restore") +{ +} + +void +thin_ll_restore_cmd::usage(ostream &out) const { + out << "Usage: " << get_name() << " [options]" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-E|--source-metadata} " << endl + << " {-i|--input} " << endl + << " {-o|--output} " << endl + << " {-V|--version}" << endl; +} + +int +thin_ll_restore_cmd::run(int argc, char **argv) { + string input; + string output; + string input_metadata; + flags f; + char c; + + const char shortopts[] = "hi:o:E:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "input", required_argument, NULL, 'i'}, + { "output", required_argument, NULL, 'o'}, + { "source-metadata", required_argument, NULL, 'E'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'i': + input = optarg; + break; + + case 'o': + output = optarg; + break; + + case 'E': + input_metadata = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr); + return 1; + } + } + + if (argc != optind) { + usage(cerr); + return 1; + } + + if (!input_metadata.length() || !input.length() || !output.length()) { + cerr << "No input/output file provided." << endl; + usage(cerr); + return 1; + } + + return low_level_restore(input_metadata, input, output, f); +} + +//--------------------------------------------------------------------------- From 3f6cae4ebcdb8945e3bd4548e9fbb6596a248ee3 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 29 Feb 2016 11:12:50 +0000 Subject: [PATCH 057/118] [persistent-data] persistent-data/data-structures/btree_key_value_extractor.h doesn't appear to be used So remove --- .../btree_key_value_extractor.h | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 persistent-data/data-structures/btree_key_value_extractor.h diff --git a/persistent-data/data-structures/btree_key_value_extractor.h b/persistent-data/data-structures/btree_key_value_extractor.h deleted file mode 100644 index 40b7f7b..0000000 --- a/persistent-data/data-structures/btree_key_value_extractor.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H -#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H - -#include "persistent-data/data-structures/btree_damage_visitor.h" -#include - -namespace persistent_data { - namespace btree_detail { - template - class key_value_extractor { - typedef typename std::map MapType; - public: - key_value_extractor(MapType &map): map_(map) { - } - - virtual ~key_value_extractor() { - } - - virtual void visit(btree_path const &path, ValueType const &v) { - map_.insert(std::make_pair(path.back(), v)); - } - private: - MapType &map_; - }; - - template - void btree_extract_key_values(btree const &tree, - std::map &map) { - typedef key_value_extractor KeyValueExtractor; - KeyValueExtractor kve(map); - noop_damage_visitor noop_dv; - btree_detail::btree_damage_visitor - v(kve, noop_dv); - tree.visit_depth_first(v); - } - } -} - -#endif From 4c0d5c96a02943e7d49e6db3664d70699ddb826a Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 29 Feb 2016 13:51:30 +0000 Subject: [PATCH 058/118] [build] add symlinks for the new dev tools --- Makefile.in | 6 ++++++ bin/thin_ll_dump | 1 + bin/thin_ll_restore | 1 + bin/thin_scan | 1 + 4 files changed, 9 insertions(+) create mode 120000 bin/thin_ll_dump create mode 120000 bin/thin_ll_restore create mode 120000 bin/thin_scan diff --git a/Makefile.in b/Makefile.in index a8c41a3..a0e7aa4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -247,6 +247,12 @@ install: bin/pdata_tools $(INSTALL_DATA) man8/era_check.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/era_dump.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/era_invalidate.8 $(MANPATH)/man8 +ifeq ("@DEVTOOLS@", "yes") + ln -s -f pdata_tools $(BINDIR)/thin_ll_dump + ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates + ln -s -f pdata_tools $(BINDIR)/thin_generate_metadata + ln -s -f pdata_tools $(BINDIR)/thin_scan +endif # $(INSTALL_DATA) man8/era_restore.8 $(MANPATH)/man8 diff --git a/bin/thin_ll_dump b/bin/thin_ll_dump new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_ll_dump @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file diff --git a/bin/thin_ll_restore b/bin/thin_ll_restore new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_ll_restore @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file diff --git a/bin/thin_scan b/bin/thin_scan new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_scan @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file From 08219a60b683c4487b8690b0dda44aaaf3b1e4a6 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 5 Mar 2016 13:59:24 +0800 Subject: [PATCH 059/118] [thin] Add default constructor to struct device_details --- thin-provisioning/device_tree.cc | 7 +++++++ thin-provisioning/device_tree.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/thin-provisioning/device_tree.cc b/thin-provisioning/device_tree.cc index 223d816..4837cb7 100644 --- a/thin-provisioning/device_tree.cc +++ b/thin-provisioning/device_tree.cc @@ -47,6 +47,13 @@ namespace { namespace thin_provisioning { namespace device_tree_detail { + device_details::device_details() + : mapped_blocks_(0), + transaction_id_(0), + creation_time_(0), + snapshotted_time_(0) { + } + void device_details_traits::unpack(device_details_disk const &disk, device_details &value) { diff --git a/thin-provisioning/device_tree.h b/thin-provisioning/device_tree.h index 23ae924..ef32b45 100644 --- a/thin-provisioning/device_tree.h +++ b/thin-provisioning/device_tree.h @@ -20,6 +20,8 @@ namespace thin_provisioning { uint64_t transaction_id_; /* when created */ uint32_t creation_time_; uint32_t snapshotted_time_; + + device_details(); }; struct device_details_traits { From 400613305c2feb490366ed383adbe4e4386772ed Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 5 Mar 2016 14:11:38 +0800 Subject: [PATCH 060/118] [thin_ll_restore] Cleanup: use device_details' constructor --- thin-provisioning/thin_ll_restore.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/thin-provisioning/thin_ll_restore.cc b/thin-provisioning/thin_ll_restore.cc index 1168579..27ab184 100644 --- a/thin-provisioning/thin_ll_restore.cc +++ b/thin-provisioning/thin_ll_restore.cc @@ -85,9 +85,6 @@ namespace { void parse_device(metadata::ptr md, emitter::ptr e, attributes const &attr) { uint32_t dev_id = get_attr(attr, "dev_id"); device_tree_detail::device_details details; - details.transaction_id_ = 0; - details.creation_time_ = 0; - details.snapshotted_time_ = 0; device_tree::ptr details_tree; boost::optional details_root = get_opt_attr(attr, "blocknr"); From 6fb5f8241d46caa32ee6e374c45670c738d1ca6d Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 5 Mar 2016 22:31:09 +0800 Subject: [PATCH 061/118] [thin_scan] Factor out range manipulation code --- thin-provisioning/thin_scan.cc | 49 +++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/thin-provisioning/thin_scan.cc b/thin-provisioning/thin_scan.cc index c5afbd9..94edd1d 100644 --- a/thin-provisioning/thin_scan.cc +++ b/thin-provisioning/thin_scan.cc @@ -151,10 +151,43 @@ namespace { value_size_(rhs.value_size_), is_valid_(rhs.is_valid_) { } + + uint64_t size() const { + return (end_ > begin_) ? (end_ - begin_) : 0; + } + + // returns true if r is left or right-adjacent + bool is_adjacent_to(block_range const &r) const { + block_range const &lhs = begin_ < r.begin_ ? *this : r; + block_range const &rhs = begin_ < r.begin_ ? r : *this; + uint64_t common_end = std::min(end_, r.end_); + + if (size() && r.size() && + rhs.begin_ == common_end && + ((!blocknr_begin_ && !r.blocknr_begin_) || + (blocknr_begin_ && r.blocknr_begin_ && + *rhs.blocknr_begin_ > *lhs.blocknr_begin_ && + (*rhs.blocknr_begin_ - *lhs.blocknr_begin_ == rhs.begin_ - lhs.begin_))) && + type_ == r.type_ && + ref_count_ == r.ref_count_ && + value_size_ == r.value_size_ && + is_valid_ == r.is_valid_) + return true; + + return false; + } + + bool concat(block_range const &r) { + if (!is_adjacent_to(r)) + return false; + begin_ = std::min(begin_, r.begin_); + end_ = std::max(end_, r.end_); + return true; + } }; void output_block_range(block_range const &r, std::ostream &out) { - if (r.end_ <= r.begin_) + if (!r.size()) return; if (r.end_ - r.begin_ > 1) { @@ -288,18 +321,8 @@ namespace { curr_range.ref_count_ = -1; } - // output the current block - if (run_range.end_ == 0) - run_range = curr_range; - else if (((!curr_range.blocknr_begin_ && !run_range.blocknr_begin_) || - (curr_range.blocknr_begin_ && run_range.blocknr_begin_ && - *curr_range.blocknr_begin_ == *run_range.blocknr_begin_ + (run_range.end_ - run_range.begin_))) && - curr_range.type_ == run_range.type_ && - curr_range.ref_count_ == run_range.ref_count_ && - curr_range.value_size_ == run_range.value_size_ && - curr_range.is_valid_ == run_range.is_valid_) { - ++run_range.end_; - } else { + // store the current block + if (!run_range.concat(curr_range)) { output_block_range(run_range, out); run_range = curr_range; } From 9f15c5589f8bb851674f97c8bad2ef6ddf637b27 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 6 Mar 2016 00:06:27 +0800 Subject: [PATCH 062/118] [thin_scan] Cleanup: move methods before data member declaration --- thin-provisioning/thin_scan.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/thin-provisioning/thin_scan.cc b/thin-provisioning/thin_scan.cc index 94edd1d..725c3b6 100644 --- a/thin-provisioning/thin_scan.cc +++ b/thin-provisioning/thin_scan.cc @@ -129,14 +129,6 @@ namespace { }; struct block_range { - uint64_t begin_; - uint64_t end_; // one-pass-the-end - boost::optional blocknr_begin_; - metadata_block_type type_; - int64_t ref_count_; // ref_count in metadata space map - size_t value_size_; // btree node only - bool is_valid_; // btree node only - block_range() : begin_(0), end_(0), type_(UNKNOWN), ref_count_(-1), @@ -184,6 +176,14 @@ namespace { end_ = std::max(end_, r.end_); return true; } + + uint64_t begin_; + uint64_t end_; // one-pass-the-end + boost::optional blocknr_begin_; + metadata_block_type type_; + int64_t ref_count_; // ref_count in metadata space map + size_t value_size_; // btree node only + bool is_valid_; }; void output_block_range(block_range const &r, std::ostream &out) { From 8232feb85554b8333ea314f7ec28c9d54d28b136 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 6 Mar 2016 01:23:04 +0800 Subject: [PATCH 063/118] [thin_scan] Cleanup: remove unnecessary variables --- thin-provisioning/thin_scan.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/thin-provisioning/thin_scan.cc b/thin-provisioning/thin_scan.cc index 725c3b6..1ed7a6b 100644 --- a/thin-provisioning/thin_scan.cc +++ b/thin-provisioning/thin_scan.cc @@ -152,13 +152,12 @@ namespace { bool is_adjacent_to(block_range const &r) const { block_range const &lhs = begin_ < r.begin_ ? *this : r; block_range const &rhs = begin_ < r.begin_ ? r : *this; - uint64_t common_end = std::min(end_, r.end_); if (size() && r.size() && - rhs.begin_ == common_end && + rhs.begin_ == lhs.end_ && ((!blocknr_begin_ && !r.blocknr_begin_) || (blocknr_begin_ && r.blocknr_begin_ && - *rhs.blocknr_begin_ > *lhs.blocknr_begin_ && + *rhs.blocknr_begin_ >= *lhs.blocknr_begin_ && (*rhs.blocknr_begin_ - *lhs.blocknr_begin_ == rhs.begin_ - lhs.begin_))) && type_ == r.type_ && ref_count_ == r.ref_count_ && From 220ece1dc8bc868cd3a143a29448b81c92320b2b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 6 Mar 2016 23:21:09 +0800 Subject: [PATCH 064/118] [thin] Cleanup: move methods before data member declaration --- thin-provisioning/device_tree.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/device_tree.h b/thin-provisioning/device_tree.h index ef32b45..ec0f9f2 100644 --- a/thin-provisioning/device_tree.h +++ b/thin-provisioning/device_tree.h @@ -16,12 +16,12 @@ namespace thin_provisioning { } __attribute__ ((packed)); struct device_details { + device_details(); + uint64_t mapped_blocks_; uint64_t transaction_id_; /* when created */ uint32_t creation_time_; uint32_t snapshotted_time_; - - device_details(); }; struct device_details_traits { From b8659853bb6460274bb06fababd205fb39fc4706 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 8 Mar 2016 15:27:22 +0000 Subject: [PATCH 065/118] [cache_writeback] stub cache_writeback --- Makefile.in | 3 + caching/cache_writeback.cc | 122 +++++++++++++++++++++++++++++++++++ caching/commands.cc | 1 + caching/commands.h | 7 ++ man8/cache_writeback.8 | 53 +++++++++++++++ thin-provisioning/commands.h | 10 +-- 6 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 caching/cache_writeback.cc create mode 100644 man8/cache_writeback.8 diff --git a/Makefile.in b/Makefile.in index a0e7aa4..87f2fd7 100644 --- a/Makefile.in +++ b/Makefile.in @@ -41,6 +41,7 @@ SOURCE=\ caching/cache_metadata_size.cc \ caching/cache_repair.cc \ caching/cache_restore.cc \ + caching/cache_writeback.cc \ caching/commands.cc \ caching/hint_array.cc \ caching/mapping_array.cc \ @@ -216,6 +217,7 @@ install: bin/pdata_tools ln -s -f pdata_tools $(BINDIR)/cache_metadata_size ln -s -f pdata_tools $(BINDIR)/cache_repair ln -s -f pdata_tools $(BINDIR)/cache_restore + ln -s -f pdata_tools $(BINDIR)/cache_writeback ln -s -f pdata_tools $(BINDIR)/thin_check ln -s -f pdata_tools $(BINDIR)/thin_delta ln -s -f pdata_tools $(BINDIR)/thin_dump @@ -235,6 +237,7 @@ install: bin/pdata_tools $(INSTALL_DATA) man8/cache_dump.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/cache_repair.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/cache_restore.8 $(MANPATH)/man8 + $(INSTALL_DATA) man8/cache_writeback.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_check.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_delta.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_dump.8 $(MANPATH)/man8 diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc new file mode 100644 index 0000000..a3d27d1 --- /dev/null +++ b/caching/cache_writeback.cc @@ -0,0 +1,122 @@ +#ifndef CACHING_CACHE_WRITEBACK_H +#define CACHING_CACHE_WRITEBACK_H + +#include "caching/commands.h" +#include "version.h" + +#include +#include +#include + +using namespace caching; +using namespace boost; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + struct flags { + using maybe_string = boost::optional; + + maybe_string metadata_dev; + maybe_string origin_dev; + maybe_string fast_dev; + }; + + int writeback(flags const &f) { + return 1; + } +} + +//---------------------------------------------------------------- + +cache_writeback_cmd::cache_writeback_cmd() + : command("cache_writeback") +{ +} + +void +cache_writeback_cmd::usage(std::ostream &out) const +{ + out << "Usage: " << get_name() << " [options]\n" + << "\t\t--metadata-device \n" + << "\t\t--origin-device \n" + << "\t\t--fast-device \n" + << "Options:\n" + << " {-h|--help}\n" + << " {-V|--version}" << endl; +} + +int +cache_writeback_cmd::run(int argc, char **argv) +{ + int c; + flags fs; + char const *short_opts = "hV"; + option const long_opts[] = { + { "metadata-device", required_argument, NULL, 0 }, + { "origin-device", required_argument, NULL, 1 }, + { "fast-device", required_argument, NULL, 2 }, + { "help", no_argument, NULL, 'h'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { + switch(c) { + case 0: + fs.metadata_dev = optarg; + break; + + case 1: + fs.origin_dev = optarg; + break; + + case 2: + fs.fast_dev = optarg; + break; + + case 'h': + usage(cout); + return 0; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr); + return 1; + } + } + + if (argc != optind) { + usage(cerr); + return 1; + } + + if (!fs.metadata_dev) { + cerr << "No metadata device provided.\n\n"; + usage(cerr); + return 1; + } + + if (!fs.origin_dev) { + cerr << "No origin device provided.\n\n"; + usage(cerr); + return 1; + } + + if (!fs.fast_dev) { + cerr << "No fast device provided.\n\n"; + usage(cerr); + return 1; + } + + return writeback(fs); + +} + +//---------------------------------------------------------------- + +#endif diff --git a/caching/commands.cc b/caching/commands.cc index 1ab79e6..b150a92 100644 --- a/caching/commands.cc +++ b/caching/commands.cc @@ -13,6 +13,7 @@ caching::register_cache_commands(application &app) app.add_cmd(command::ptr(new cache_metadata_size_cmd)); app.add_cmd(command::ptr(new cache_restore_cmd)); app.add_cmd(command::ptr(new cache_repair_cmd)); + app.add_cmd(command::ptr(new cache_writeback_cmd)); } //---------------------------------------------------------------- diff --git a/caching/commands.h b/caching/commands.h index 022ac06..b0546ab 100644 --- a/caching/commands.h +++ b/caching/commands.h @@ -63,6 +63,13 @@ namespace caching { virtual int run(int argc, char **argv); }; + class cache_writeback_cmd : public base::command { + public: + cache_writeback_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + void register_cache_commands(base::application &app); } diff --git a/man8/cache_writeback.8 b/man8/cache_writeback.8 new file mode 100644 index 0000000..525b63a --- /dev/null +++ b/man8/cache_writeback.8 @@ -0,0 +1,53 @@ +.TH CACHE_WRITEBACK 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*- +.SH NAME +cache_writeback \- writeback dirty blocks to the origin device. + +.SH SYNOPSIS +.B cache_writeback +.RB [ options ] +.RB --metadata-device +.I {device|file} +.RB --origin-device +.I {device|file} +.RB --fast-device +.I {device|file} + +.SH DESCRIPTION +.B cache_writeback + +An offline tool that writesback dirty data to the data device +(origin). Intended for use in recovery scenarios when the SSD is +giving IO errors. + +This tool cannot be run on a live cache. + +.SH OPTIONS + +.IP "\fB\\-\-metadata\-device\fP \fI{device|file}\fP" +Location of cache metadata. + +.IP "\fB\-\-origin\-device\fP \fI{device|file}\fP" +Slow device being cached. + +.IP "\fB\-\-fast\-device\fP \fI{device|file}\fP" +Fast device containing the data that needs to be written back. + +.IP "\fB\-\-skip\-metadata\-update\fP" +Do not update the metadata to clear the dirty flags for written back +data. You may not want to do this if you're decommissioning the +cache. + +.IP "\fB\-h, \-\-help\fP" +Print help and exit. + +.IP "\fB\-V, \-\-version\fP" +Output version information and exit. + +.SH SEE ALSO +.B cache_dump(8) +.B cache_check(8) +.B cache_repair(8) +.B cache_restore(8) + +.SH AUTHOR +Joe Thornber diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index 1cff88f..927773c 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -77,17 +77,17 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; -#ifdef DEV_TOOLS - class thin_scan_cmd : public base::command { + class thin_trim_cmd : public base::command { public: - thin_scan_cmd(); + thin_trim_cmd(); virtual void usage(std::ostream &out) const; virtual int run(int argc, char **argv); }; - class thin_trim_cmd : public base::command { +#ifdef DEV_TOOLS + class thin_scan_cmd : public base::command { public: - thin_trim_cmd(); + thin_scan_cmd(); virtual void usage(std::ostream &out) const; virtual int run(int argc, char **argv); }; From 5cbef4f6efa0eb4565698dd41d54d175d81faf0b Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 8 Mar 2016 15:52:12 +0000 Subject: [PATCH 066/118] [thin_ll_*] move these to dev tools --- thin-provisioning/commands.cc | 4 ++-- thin-provisioning/commands.h | 38 +++++++++++++++++------------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index f4a96fd..4f2af9d 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -11,8 +11,6 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_check_cmd())); app.add_cmd(command::ptr(new thin_delta_cmd())); app.add_cmd(command::ptr(new thin_dump_cmd())); - app.add_cmd(command::ptr(new thin_ll_dump_cmd())); - app.add_cmd(command::ptr(new thin_ll_restore_cmd())); app.add_cmd(command::ptr(new thin_ls_cmd())); app.add_cmd(command::ptr(new thin_metadata_size_cmd())); app.add_cmd(command::ptr(new thin_restore_cmd())); @@ -20,6 +18,8 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_rmap_cmd())); #ifdef DEV_TOOLS + app.add_cmd(command::ptr(new thin_ll_dump_cmd())); + app.add_cmd(command::ptr(new thin_ll_restore_cmd())); app.add_cmd(command::ptr(new thin_scan_cmd())); app.add_cmd(command::ptr(new thin_trim_cmd())); app.add_cmd(command::ptr(new thin_generate_metadata_cmd())); diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index 1cff88f..52aeaf8 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -28,20 +28,6 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; - class thin_ll_dump_cmd : public base::command { - public: - thin_ll_dump_cmd(); - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; - - class thin_ll_restore_cmd : public base::command { - public: - thin_ll_restore_cmd(); - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; - class thin_ls_cmd : public base::command { public: thin_ls_cmd(); @@ -77,17 +63,31 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; -#ifdef DEV_TOOLS - class thin_scan_cmd : public base::command { + class thin_trim_cmd : public base::command { public: - thin_scan_cmd(); + thin_trim_cmd(); virtual void usage(std::ostream &out) const; virtual int run(int argc, char **argv); }; - class thin_trim_cmd : public base::command { +#ifdef DEV_TOOLS + class thin_ll_dump_cmd : public base::command { public: - thin_trim_cmd(); + thin_ll_dump_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_ll_restore_cmd : public base::command { + public: + thin_ll_restore_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_scan_cmd : public base::command { + public: + thin_scan_cmd(); virtual void usage(std::ostream &out) const; virtual int run(int argc, char **argv); }; From 4573ebb218ce9e3b7442f07a4639cec0be49d63a Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 17 Mar 2016 15:15:52 +0000 Subject: [PATCH 067/118] [cache_writeback] work in progress --- block-cache/copier.cc | 7 +++ block-cache/copier.h | 30 +++++++++ caching/cache_writeback.cc | 122 +++++++++++++++++++++++++++++++++++-- 3 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 block-cache/copier.cc create mode 100644 block-cache/copier.h diff --git a/block-cache/copier.cc b/block-cache/copier.cc new file mode 100644 index 0000000..60ca93b --- /dev/null +++ b/block-cache/copier.cc @@ -0,0 +1,7 @@ +#include "block-cache/copier.h" + +//---------------------------------------------------------------- + + + +//---------------------------------------------------------------- diff --git a/block-cache/copier.h b/block-cache/copier.h new file mode 100644 index 0000000..41ed82e --- /dev/null +++ b/block-cache/copier.h @@ -0,0 +1,30 @@ +#ifndef BLOCK_CACHE_COPIER_H +#define BLOCK_CACHE_COPIER_H + +#include "block_cache.h" + +#include + +//---------------------------------------------------------------- + +namespace bcache { + class copier { + public: + // block size in sectors + copier(std::string const &src, std::string const &dest, + unsigned block_size); + ~copier(); + + // Returns the number of sectors copied + unsigned copy(block_address from, block_address to); + + unsigned get_block_size() const; + + private: + unsigned block_size_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc index a3d27d1..042ae74 100644 --- a/caching/cache_writeback.cc +++ b/caching/cache_writeback.cc @@ -1,13 +1,16 @@ -#ifndef CACHING_CACHE_WRITEBACK_H -#define CACHING_CACHE_WRITEBACK_H - +#include "persistent-data/file_utils.h" +#include "block-cache/copier.h" #include "caching/commands.h" +#include "caching/mapping_array.h" +#include "caching/metadata.h" #include "version.h" #include #include #include +#include +using namespace bcache; using namespace caching; using namespace boost; using namespace std; @@ -23,8 +26,117 @@ namespace { maybe_string fast_dev; }; + class copy_visitor : public mapping_visitor { + public: + copy_visitor(copier &c, bool only_dirty) + : copier_(c), + block_size_(c.get_block_size()), + only_dirty_(only_dirty) { + } + + virtual void visit(block_address cblock, mapping const &m) { + if (!(m.flags_ & M_VALID)) + return; + + if (only_dirty_ && !(m.flags_ & M_DIRTY)) + return; + + auto sectors_copied = copier_.copy(cblock, m.oblock_); + + stats_.blocks_issued++; + if (sectors_copied < block_size_) { + stats_.blocks_failed++; + stats_.sectors_failed += block_size_ - sectors_copied; + } + } + + struct copy_stats { + copy_stats() + : blocks_issued(0), + blocks_failed(0), + sectors_failed(0) { + } + + block_address blocks_issued; + block_address blocks_failed; + block_address sectors_failed; + }; + + copy_stats const &get_stats() const { + return stats_; + } + + private: + copier &copier_; + unsigned block_size_; + bool only_dirty_; + copy_stats stats_; + }; + + using namespace mapping_array_damage; + + class ignore_damage_visitor : public damage_visitor { + public: + ignore_damage_visitor() + : corruption_(false) { + } + + void visit(missing_mappings const &d) { + cerr << "missing mappings (" << d.keys_.begin_ << ", " << d.keys_.end_ << "]\n"; + corruption_ = true; + } + + void visit(invalid_mapping const &d) { + cerr << "invalid mapping cblock = " << d.cblock_ << ", oblock = " << d.m_.oblock_ << "\n"; + corruption_ = true; + } + + bool was_corruption() const { + return corruption_; + } + + private: + bool corruption_; + }; + + bool clean_shutdown(metadata const &md) { + return md.sb_.flags.get_flag(superblock_flags::CLEAN_SHUTDOWN); + } + + int writeback_(flags const &f) { + block_manager<>::ptr bm = open_bm(*f.metadata_dev, block_manager<>::READ_ONLY); + metadata md(bm, metadata::OPEN); + + copier c(*f.fast_dev, *f.origin_dev, md.sb_.data_block_size); + copy_visitor cv(c, clean_shutdown(md)); + ignore_damage_visitor dv; + walk_mapping_array(*md.mappings_, cv, dv); + + auto stats = cv.get_stats(); + cout << stats.blocks_issued << " copies issued\n" + << stats.blocks_failed << " copies failed\n"; + + if (stats.blocks_failed) + cout << stats.sectors_failed << " sectors were not copied\n"; + + if (dv.was_corruption()) + cout << "Metadata corruption was found, some data may not have been copied.\n"; + + return (stats.blocks_failed || dv.was_corruption()) ? 1 : 0; + } + int writeback(flags const &f) { - return 1; + int r; + + try { + r = writeback_(f); + + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + + return r; } } @@ -118,5 +230,3 @@ cache_writeback_cmd::run(int argc, char **argv) } //---------------------------------------------------------------- - -#endif From 055623b90f7ff7b65c2b1b15ec142d41ebb93708 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 19 Mar 2016 13:59:38 +0800 Subject: [PATCH 068/118] [thin_ll_dump] Fix option -m to accept optional argument --- thin-provisioning/thin_ll_dump.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thin-provisioning/thin_ll_dump.cc b/thin-provisioning/thin_ll_dump.cc index 7f7b14c..2a20237 100644 --- a/thin-provisioning/thin_ll_dump.cc +++ b/thin-provisioning/thin_ll_dump.cc @@ -371,7 +371,7 @@ thin_ll_dump_cmd::usage(ostream &out) const { int thin_ll_dump_cmd::run(int argc, char **argv) { - const char shortopts[] = "hm:o:V"; + const char shortopts[] = "hm::o:V"; const struct option longopts[] = { { "help", no_argument, NULL, 'h'}, { "metadata-snap", optional_argument, NULL, 'm'}, From a3f4940f1e7cd8bccea5af3a2dfe4abcaf3c160d Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 22 Mar 2016 14:41:02 +0000 Subject: [PATCH 069/118] [thin_dump] --dev-id --- man8/thin_dump.8 | 4 +++ thin-provisioning/metadata_dumper.cc | 33 +++++++++++++++--------- thin-provisioning/metadata_dumper.h | 29 ++++++++++++++++++++- thin-provisioning/thin_dump.cc | 38 ++++++++++++++++++---------- thin-provisioning/thin_repair.cc | 5 +++- 5 files changed, 82 insertions(+), 27 deletions(-) diff --git a/man8/thin_dump.8 b/man8/thin_dump.8 index 7a9f785..4d8c1a8 100644 --- a/man8/thin_dump.8 +++ b/man8/thin_dump.8 @@ -39,6 +39,10 @@ the thin provisioning device-mapper target, else try the one at block#. See the thin provisioning target documentation on how to create or release a metadata snapshot and retrieve the block number from the kernel. +.IP "\fN\-\-dev\-id\fP ". +Dump the specified device. This option may be specified multiple +times to select more than one thin device. + .IP "\fB\-h, \-\-help\fP". Print help and exit. diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index 0cca245..96f64a3 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -88,8 +88,13 @@ namespace { class details_extractor : public device_tree_detail::device_visitor { public: + details_extractor(dump_options const &opts) + : opts_(opts) { + } + void visit(block_address dev_id, device_tree_detail::device_details const &dd) { - dd_.insert(make_pair(dev_id, dd)); + if (opts_.selected_dev(dev_id)) + dd_.insert(make_pair(dev_id, dd)); } dd_map const &get_details() const { @@ -97,6 +102,7 @@ namespace { } private: + dump_options const &opts_; dd_map dd_; }; @@ -161,21 +167,24 @@ namespace { class mapping_tree_emitter : public mapping_tree_detail::device_visitor { public: - mapping_tree_emitter(metadata::ptr md, + mapping_tree_emitter(dump_options const &opts, + metadata::ptr md, emitter::ptr e, dd_map const &dd, - bool repair, mapping_tree_detail::damage_visitor::ptr damage_policy) - : md_(md), + : opts_(opts), + md_(md), e_(e), dd_(dd), - repair_(repair), damage_policy_(damage_policy) { } void visit(btree_path const &path, block_address tree_root) { block_address dev_id = path[0]; + if (!opts_.selected_dev(dev_id)) + return; + dd_map::const_iterator it = dd_.find(path[0]); if (it != dd_.end()) { device_tree_detail::device_details const &d = it->second; @@ -194,7 +203,7 @@ namespace { } e_->end_device(); - } else if (!repair_) { + } else if (!opts_.repair_) { ostringstream msg; msg << "mappings present for device " << dev_id << ", but it isn't present in device tree"; @@ -210,10 +219,10 @@ namespace { walk_mapping_tree(tree, static_cast(me), *damage_policy_); } + dump_options const &opts_; metadata::ptr md_; emitter::ptr e_; dd_map const &dd_; - bool repair_; mapping_tree_detail::damage_visitor::ptr damage_policy_; }; @@ -234,10 +243,10 @@ namespace { //---------------------------------------------------------------- void -thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) +thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, dump_options const &opts) { - details_extractor de; - device_tree_detail::damage_visitor::ptr dd_policy(details_damage_policy(repair)); + details_extractor de(opts); + device_tree_detail::damage_visitor::ptr dd_policy(details_damage_policy(opts.repair_)); walk_device_tree(*md->details_, de, *dd_policy); e->begin_superblock("", md->sb_.time_, @@ -249,8 +258,8 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) boost::optional()); { - mapping_tree_detail::damage_visitor::ptr md_policy(mapping_damage_policy(repair)); - mapping_tree_emitter mte(md, e, de.get_details(), repair, mapping_damage_policy(repair)); + mapping_tree_detail::damage_visitor::ptr md_policy(mapping_damage_policy(opts.repair_)); + mapping_tree_emitter mte(opts, md, e, de.get_details(), mapping_damage_policy(opts.repair_)); walk_mapping_tree(*md->mappings_top_level_, mte, *md_policy); } diff --git a/thin-provisioning/metadata_dumper.h b/thin-provisioning/metadata_dumper.h index 007256a..a615a9e 100644 --- a/thin-provisioning/metadata_dumper.h +++ b/thin-provisioning/metadata_dumper.h @@ -22,13 +22,40 @@ #include "emitter.h" #include "metadata.h" +#include +#include + //---------------------------------------------------------------- namespace thin_provisioning { + class dump_options { + public: + dump_options() + : repair_(false) { + } + + bool selected_dev(uint64_t dev_id) const { + return !dev_filter_ || dev_filter_->count(dev_id); + } + + void select_dev(uint64_t dev_id) { + if (!dev_filter_) + dev_filter_ = dev_set(); + + dev_filter_->insert(dev_id); + } + + bool repair_; + + using dev_set = std::set; + using maybe_dev_set = boost::optional; + maybe_dev_set dev_filter_; + }; + // Set the @repair flag if your metadata is corrupt, and you'd like // the dumper to do it's best to recover info. If not set, any // corruption encountered will cause an exception to be thrown. - void metadata_dump(metadata::ptr md, emitter::ptr e, bool repair); + void metadata_dump(metadata::ptr md, emitter::ptr e, dump_options const &opts); void metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root); } diff --git a/thin-provisioning/thin_dump.cc b/thin-provisioning/thin_dump.cc index 171c263..515b31a 100644 --- a/thin-provisioning/thin_dump.cc +++ b/thin-provisioning/thin_dump.cc @@ -38,11 +38,10 @@ namespace { // FIXME: put the path into the flags struct flags { flags() - : repair(false), - use_metadata_snap(false) { + : use_metadata_snap(false) { } - bool repair; + dump_options opts; bool use_metadata_snap; optional snap_location; }; @@ -70,7 +69,7 @@ namespace { exit(1); } - metadata_dump(md, e, flags.repair); + metadata_dump(md, e, flags.opts); } catch (std::exception &e) { cerr << e.what() << endl; @@ -99,13 +98,14 @@ thin_dump_cmd::thin_dump_cmd() void thin_dump_cmd::usage(std::ostream &out) const { - out << "Usage: " << get_name() << " [options] {device|file}" << endl - << "Options:" << endl - << " {-h|--help}" << endl - << " {-f|--format} {xml|human_readable}" << endl - << " {-r|--repair}" << endl - << " {-m|--metadata-snap} [block#]" << endl - << " {-o }" << endl + out << "Usage: " << get_name() << " [options] {device|file}\n" + << "Options:\n" + << " {-h|--help}\n" + << " {-f|--format} {xml|human_readable}\n" + << " {-r|--repair}\n" + << " {-m|--metadata-snap} [block#]\n" + << " {-o }\n" + << " {--dev-id} \n" << " {-V|--version}" << endl; } @@ -118,6 +118,7 @@ thin_dump_cmd::run(int argc, char **argv) char *end_ptr; string format = "xml"; block_address metadata_snap = 0; + uint64_t dev_id; struct flags flags; const struct option longopts[] = { @@ -126,6 +127,7 @@ thin_dump_cmd::run(int argc, char **argv) { "output", required_argument, NULL, 'o'}, { "format", required_argument, NULL, 'f' }, { "repair", no_argument, NULL, 'r'}, + { "dev-id", required_argument, NULL, 1 }, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } }; @@ -141,7 +143,7 @@ thin_dump_cmd::run(int argc, char **argv) break; case 'r': - flags.repair = true; + flags.opts.repair_ = true; break; case 'm': @@ -150,7 +152,7 @@ thin_dump_cmd::run(int argc, char **argv) // FIXME: deprecate this option metadata_snap = strtoull(optarg, &end_ptr, 10); if (end_ptr == optarg) { - cerr << "couldn't parse " << endl; + cerr << "couldn't parse " << endl; usage(cerr); return 1; } @@ -163,6 +165,16 @@ thin_dump_cmd::run(int argc, char **argv) output = optarg; break; + case 1: + dev_id = strtoull(optarg, &end_ptr, 10); + if (end_ptr == optarg) { + cerr << "couldn't parse \n"; + usage(cerr); + return 1; + } + flags.opts.select_dev(dev_id); + break; + case 'V': cout << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; diff --git a/thin-provisioning/thin_repair.cc b/thin-provisioning/thin_repair.cc index 2741b78..fe0f010 100644 --- a/thin-provisioning/thin_repair.cc +++ b/thin-provisioning/thin_repair.cc @@ -25,7 +25,10 @@ namespace { block_manager<>::ptr old_bm = open_bm(old_path, block_manager<>::READ_ONLY); metadata::ptr old_md(new metadata(old_bm, false)); - metadata_dump(old_md, e, true); + + dump_options opts; + opts.repair_ = true; + metadata_dump(old_md, e, opts); } catch (std::exception &e) { cerr << e.what() << endl; From 4370f048c062622a2b9edff1cc0d66ab496b949f Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 22 Mar 2016 15:01:37 +0000 Subject: [PATCH 070/118] [thin_dump] --skip-mappings --- man8/thin_dump.8 | 5 ++++- thin-provisioning/metadata_dumper.cc | 3 ++- thin-provisioning/metadata_dumper.h | 4 +++- thin-provisioning/thin_dump.cc | 5 +++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/man8/thin_dump.8 b/man8/thin_dump.8 index 4d8c1a8..28b6d8b 100644 --- a/man8/thin_dump.8 +++ b/man8/thin_dump.8 @@ -39,10 +39,13 @@ the thin provisioning device-mapper target, else try the one at block#. See the thin provisioning target documentation on how to create or release a metadata snapshot and retrieve the block number from the kernel. -.IP "\fN\-\-dev\-id\fP ". +.IP "\fB\-\-dev\-id\fP ". Dump the specified device. This option may be specified multiple times to select more than one thin device. +.IP "\fB\-\-skip\-mappings". +Do not dump the mappings. + .IP "\fB\-h, \-\-help\fP". Print help and exit. diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index 96f64a3..ba29bfa 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -195,7 +195,8 @@ namespace { d.snapshotted_time_); try { - emit_mappings(tree_root); + if (!opts_.skip_mappings_) + emit_mappings(tree_root); } catch (exception &e) { cerr << e.what(); e_->end_device(); diff --git a/thin-provisioning/metadata_dumper.h b/thin-provisioning/metadata_dumper.h index a615a9e..15bf668 100644 --- a/thin-provisioning/metadata_dumper.h +++ b/thin-provisioning/metadata_dumper.h @@ -31,7 +31,8 @@ namespace thin_provisioning { class dump_options { public: dump_options() - : repair_(false) { + : repair_(false), + skip_mappings_(false) { } bool selected_dev(uint64_t dev_id) const { @@ -46,6 +47,7 @@ namespace thin_provisioning { } bool repair_; + bool skip_mappings_; using dev_set = std::set; using maybe_dev_set = boost::optional; diff --git a/thin-provisioning/thin_dump.cc b/thin-provisioning/thin_dump.cc index 515b31a..932fa3a 100644 --- a/thin-provisioning/thin_dump.cc +++ b/thin-provisioning/thin_dump.cc @@ -128,6 +128,7 @@ thin_dump_cmd::run(int argc, char **argv) { "format", required_argument, NULL, 'f' }, { "repair", no_argument, NULL, 'r'}, { "dev-id", required_argument, NULL, 1 }, + { "skip-mappings", no_argument, NULL, 2 }, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } }; @@ -175,6 +176,10 @@ thin_dump_cmd::run(int argc, char **argv) flags.opts.select_dev(dev_id); break; + case 2: + flags.opts.skip_mappings_ = true; + break; + case 'V': cout << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; From 872a9330728c13ddbffb57e01bb629df58749bc6 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 23 Mar 2016 10:52:04 +0000 Subject: [PATCH 071/118] [thin_dump] start refactoring format code --- thin-provisioning/thin_dump.cc | 51 +++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/thin-provisioning/thin_dump.cc b/thin-provisioning/thin_dump.cc index 932fa3a..87d78c0 100644 --- a/thin-provisioning/thin_dump.cc +++ b/thin-provisioning/thin_dump.cc @@ -38,10 +38,13 @@ namespace { // FIXME: put the path into the flags struct flags { flags() - : use_metadata_snap(false) { + : format("xml"), + use_metadata_snap(false) { } dump_options opts; + + string format; bool use_metadata_snap; optional snap_location; }; @@ -53,22 +56,27 @@ namespace { return md; } - int dump_(string const &path, ostream &out, string const &format, struct flags &flags) { + emitter::ptr create_emitter(string const &format, ostream &out) { + emitter::ptr e; + + if (format == "xml") + e = create_xml_emitter(out); + + else if (format == "human_readable") + e = create_human_readable_emitter(out); + + else { + cerr << "unknown format '" << format << "'" << endl; + exit(1); + } + + return e; + } + + int dump_(string const &path, ostream &out, struct flags &flags) { try { metadata::ptr md = open_metadata(path, flags); - emitter::ptr e; - - if (format == "xml") - e = create_xml_emitter(out); - - else if (format == "human_readable") - e = create_human_readable_emitter(out); - - else { - cerr << "unknown format '" << format << "'" << endl; - exit(1); - } - + emitter::ptr e = create_emitter(flags.format, out); metadata_dump(md, e, flags.opts); } catch (std::exception &e) { @@ -79,12 +87,12 @@ namespace { return 0; } - int dump(string const &path, char const *output, string const &format, struct flags &flags) { + int dump(string const &path, char const *output, struct flags &flags) { if (output) { ofstream out(output); - return dump_(path, out, format, flags); + return dump_(path, out, flags); } else - return dump_(path, cout, format, flags); + return dump_(path, cout, flags); } } @@ -101,7 +109,7 @@ thin_dump_cmd::usage(std::ostream &out) const out << "Usage: " << get_name() << " [options] {device|file}\n" << "Options:\n" << " {-h|--help}\n" - << " {-f|--format} {xml|human_readable}\n" + << " {-f|--format} {xml|human_readable|custom}\n" << " {-r|--repair}\n" << " {-m|--metadata-snap} [block#]\n" << " {-o }\n" @@ -116,7 +124,6 @@ thin_dump_cmd::run(int argc, char **argv) char const *output = NULL; const char shortopts[] = "hm::o:f:rV"; char *end_ptr; - string format = "xml"; block_address metadata_snap = 0; uint64_t dev_id; struct flags flags; @@ -140,7 +147,7 @@ thin_dump_cmd::run(int argc, char **argv) return 0; case 'f': - format = optarg; + flags.format = optarg; break; case 'r': @@ -196,7 +203,7 @@ thin_dump_cmd::run(int argc, char **argv) return 1; } - return dump(argv[optind], output, format, flags); + return dump(argv[optind], output, flags); } //---------------------------------------------------------------- From c7813e07e4d2b2d64120096bd8e9421d0dcf6083 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 24 Mar 2016 13:10:37 +0000 Subject: [PATCH 072/118] [thin_dump] --format custom= Allow people to use their own emitters held in a shared library. Put a trivial emitter in contrib/ as an example. --- Makefile.in | 6 +- configure.ac | 1 + contrib/Makefile.in | 7 ++ contrib/thin_sexp_emitter.cc | 120 ++++++++++++++++++++ man8/thin_dump.8 | 9 +- thin-provisioning/shared_library_emitter.cc | 29 +++++ thin-provisioning/shared_library_emitter.h | 14 +++ thin-provisioning/thin_dump.cc | 25 ++-- 8 files changed, 199 insertions(+), 12 deletions(-) create mode 100644 contrib/Makefile.in create mode 100644 contrib/thin_sexp_emitter.cc create mode 100644 thin-provisioning/shared_library_emitter.cc create mode 100644 thin-provisioning/shared_library_emitter.h diff --git a/Makefile.in b/Makefile.in index a0e7aa4..8af54a0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -91,6 +91,7 @@ SOURCE=\ thin-provisioning/restore_emitter.cc \ thin-provisioning/rmap_visitor.cc \ thin-provisioning/superblock.cc \ + thin-provisioning/shared_library_emitter.cc \ thin-provisioning/thin_check.cc \ thin-provisioning/thin_delta.cc \ thin-provisioning/thin_dump.cc \ @@ -123,9 +124,9 @@ STRIP:=@STRIP@ OBJECTS:=$(subst .cc,.o,$(SOURCE)) TOP_DIR:=@top_srcdir@ TOP_BUILDDIR:=@top_builddir@ -CFLAGS+=-g -Wall -O3 +CFLAGS+=-g -Wall -O3 -fPIC CFLAGS+=@LFS_FLAGS@ -CXXFLAGS+=-g -Wall -fno-strict-aliasing -std=c++11 +CXXFLAGS+=-g -Wall -fPIC -fno-strict-aliasing -std=c++11 ifeq ("@DEVTOOLS@", "yes") CXXFLAGS+=-DDEV_TOOLS @@ -260,6 +261,7 @@ endif ifeq ("@TESTING@", "yes") include unit-tests/Makefile +include contrib/Makefile .PHONEY: features diff --git a/configure.ac b/configure.ac index d9600b9..9b0b2ce 100644 --- a/configure.ac +++ b/configure.ac @@ -188,6 +188,7 @@ dnl -- First and last lines should not contain files to generate in order to dnl -- keep utility scripts running properly AC_CONFIG_FILES([ Makefile +contrib/Makefile unit-tests/Makefile version.h ]) diff --git a/contrib/Makefile.in b/contrib/Makefile.in new file mode 100644 index 0000000..a47c8e2 --- /dev/null +++ b/contrib/Makefile.in @@ -0,0 +1,7 @@ +contrib: contrib/thin_sexp_emitter.so + +contrib/thin_sexp_emitter.so: contrib/thin_sexp_emitter.o + $(V)echo " [LD] $@" + $(V)$(CC) -shared -Wl,-soname,thin_sexp_emitter.so -o contrib/thin_sexp_emitter.so contrib/thin_sexp_emitter.o + + diff --git a/contrib/thin_sexp_emitter.cc b/contrib/thin_sexp_emitter.cc new file mode 100644 index 0000000..281f0ec --- /dev/null +++ b/contrib/thin_sexp_emitter.cc @@ -0,0 +1,120 @@ +#include "base/indented_stream.h" +#include "thin-provisioning/emitter.h" + +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + class sexp_emitter : public emitter { + public: + sexp_emitter(ostream &out) + : out_(out) { + } + + virtual void begin_superblock(std::string const &uuid, + uint64_t time, + uint64_t trans_id, + boost::optional flags, + boost::optional version, + uint32_t data_block_size, + uint64_t nr_data_blocks, + boost::optional metadata_snap) { + open("superblock"); + out_.indent(); + out_ << "((uuid \"" << uuid << "\")\n"; + kv("time", time); + kv("trans_id", trans_id); + kv("flags", flags); + kv("version", version); + kv("data_block_size", data_block_size); + kv("nr_data_blocks", nr_data_blocks); + kv("metadata_snap", metadata_snap); + out_.indent(); + out_ << ")\n"; + } + + virtual void end_superblock() { + close(); + } + + virtual void begin_device(uint32_t dev_id, + uint64_t mapped_blocks, + uint64_t trans_id, + uint64_t creation_time, + uint64_t snap_time) { + open("device"); + out_.indent(); + out_ << "((dev_id " << dev_id << ")\n"; + kv("mapped_blocks", mapped_blocks); + kv("trans_id", trans_id); + kv("creation_time", creation_time); + kv("snap_time", snap_time); + out_.indent(); + out_ << ")\n"; + } + + virtual void end_device() { + close(); + } + + virtual void begin_named_mapping(std::string const &name) { + + } + + virtual void end_named_mapping() { + + } + + virtual void identifier(std::string const &name) { + + } + + virtual void range_map(uint64_t origin_begin, uint64_t data_begin, uint32_t time, uint64_t len) { + out_.indent(); + out_ << "(range (origin_begin " << origin_begin + << ") (data_begin " << data_begin + << ") (time " << time + << ") (len " << len << "))\n"; + } + + virtual void single_map(uint64_t origin_block, uint64_t data_block, uint32_t time) { + out_.indent(); + out_ << "(single (origin_block " << origin_block + << ") (data_block " << data_block + << ") (time " << time << "))\n"; + } + + private: + void open(char const *tag) { + out_.indent(); + out_ << "(" << tag << "\n"; + out_.inc(); + } + + void close() { + out_.dec(); + out_.indent(); + out_ << ")\n"; + } + + template + void kv(char const *k, T const &v) { + out_.indent(); + out_ << " (" << k << " " << v << ")\n"; + } + + indented_stream out_; + }; +} + +//---------------------------------------------------------------- + +extern "C" { + emitter::ptr create_emitter(ostream &out) { + return emitter::ptr(new sexp_emitter(out)); + } +} + +//---------------------------------------------------------------- diff --git a/man8/thin_dump.8 b/man8/thin_dump.8 index 28b6d8b..eb81c29 100644 --- a/man8/thin_dump.8 +++ b/man8/thin_dump.8 @@ -26,8 +26,13 @@ in order to put it back onto a metadata This tool cannot be run on live metadata unless the \fB\-\-metadata\-snap\fP option is used. -.IP "\fB\-f, \-\-format\fP \fI{xml|human_readable}\fP". -Print output in XML or human readable format. +.IP "\fB\-f, \-\-format\fP \fI{xml|human_readable|custom}\fP". + +Print output in XML or human readable format. Custom formats are +supported via shared library plugins. They should be specified as in +this example: +.sp +.B thin_dump --format custom=mylib.so /dev/sda .IP "\fB\-r, \-\-repair\fP". Repair the metadata whilst dumping it. diff --git a/thin-provisioning/shared_library_emitter.cc b/thin-provisioning/shared_library_emitter.cc new file mode 100644 index 0000000..58f12d2 --- /dev/null +++ b/thin-provisioning/shared_library_emitter.cc @@ -0,0 +1,29 @@ +#include "thin-provisioning/shared_library_emitter.h" + +#include +#include + +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +emitter::ptr +thin_provisioning::create_custom_emitter(string const &shared_lib, ostream &out) +{ + emitter::ptr (*create_fn)(ostream &out); + void *handle = dlopen(shared_lib.c_str(), RTLD_LAZY); + if (!handle) + throw runtime_error(dlerror()); + + dlerror(); // Clear any existing error + create_fn = reinterpret_cast(dlsym(handle, "create_emitter")); + + char *error = dlerror(); + if (error) + throw runtime_error(error); + + return create_fn(out); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/shared_library_emitter.h b/thin-provisioning/shared_library_emitter.h new file mode 100644 index 0000000..e0f8b95 --- /dev/null +++ b/thin-provisioning/shared_library_emitter.h @@ -0,0 +1,14 @@ +#ifndef THIN_PROVISIONING_SHARED_LIBRARY_EMITTER_H +#define THIN_PROVISIONING_SHARED_LIBRARY_EMITTER_H + +#include "thin-provisioning/emitter.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + emitter::ptr create_custom_emitter(std::string const &shared_lib, std::ostream &out); +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/thin_dump.cc b/thin-provisioning/thin_dump.cc index 87d78c0..c6a964a 100644 --- a/thin-provisioning/thin_dump.cc +++ b/thin-provisioning/thin_dump.cc @@ -21,13 +21,14 @@ #include #include -#include "human_readable_format.h" -#include "metadata_dumper.h" -#include "metadata.h" -#include "xml_format.h" -#include "version.h" -#include "thin-provisioning/commands.h" #include "persistent-data/file_utils.h" +#include "thin-provisioning/commands.h" +#include "thin-provisioning/human_readable_format.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/metadata_dumper.h" +#include "thin-provisioning/shared_library_emitter.h" +#include "thin-provisioning/xml_format.h" +#include "version.h" using namespace boost; using namespace persistent_data; @@ -56,6 +57,10 @@ namespace { return md; } + bool begins_with(string const &str, string const &prefix) { + return str.substr(0, prefix.length()) == prefix; + } + emitter::ptr create_emitter(string const &format, ostream &out) { emitter::ptr e; @@ -65,9 +70,13 @@ namespace { else if (format == "human_readable") e = create_human_readable_emitter(out); + else if (begins_with(format, "custom=")) + e = create_custom_emitter(format.substr(7), out); + else { - cerr << "unknown format '" << format << "'" << endl; - exit(1); + ostringstream msg; + msg << "unknown format '" << format << "'"; + throw runtime_error(msg.str()); } return e; From 6c4f3ad2ff45567023dcdfd73b2c0cc66fa51fdb Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 24 Mar 2016 14:44:13 +0000 Subject: [PATCH 073/118] [contrib/ tmakatos_emitter for thin_dump --- contrib/Makefile.in | 12 ++- contrib/tmakatos_emitter.cc | 151 ++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 contrib/tmakatos_emitter.cc diff --git a/contrib/Makefile.in b/contrib/Makefile.in index a47c8e2..58cfefc 100644 --- a/contrib/Makefile.in +++ b/contrib/Makefile.in @@ -1,7 +1,15 @@ -contrib: contrib/thin_sexp_emitter.so +PLUGINS=\ + contrib/thin_sexp_emitter.so \ + contrib/tmakatos_emitter.so + +contrib: $(PLUGINS) contrib/thin_sexp_emitter.so: contrib/thin_sexp_emitter.o $(V)echo " [LD] $@" - $(V)$(CC) -shared -Wl,-soname,thin_sexp_emitter.so -o contrib/thin_sexp_emitter.so contrib/thin_sexp_emitter.o + $(V)$(CC) -shared -Wl,-soname,thin_sexp_emitter.so -o $@ $< + +contrib/tmakatos_emitter.so: contrib/tmakatos_emitter.o + $(V)echo " [LD] $@" + $(V)$(CC) -shared -Wl,-soname,tmakatos_emitter.so -o $@ $< diff --git a/contrib/tmakatos_emitter.cc b/contrib/tmakatos_emitter.cc new file mode 100644 index 0000000..4699077 --- /dev/null +++ b/contrib/tmakatos_emitter.cc @@ -0,0 +1,151 @@ +#include "thin-provisioning/emitter.h" + +#include +#include + +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + template + std::ostream &operator << (ostream &out, boost::optional const &maybe) { + if (maybe) + out << *maybe; + + return out; + } + + //------------------------------------------------ + // binary generator + //------------------------------------------------ + class binary_emitter : public emitter { + public: + binary_emitter(ostream &out) + : out_(out) { + } + + void begin_superblock(string const &uuid, + uint64_t time, + uint64_t trans_id, + boost::optional flags, + boost::optional version, + uint32_t data_block_size, + uint64_t nr_data_blocks, + boost::optional metadata_snap) { + } + + void end_superblock() { + } + + void begin_device(uint32_t dev_id, + uint64_t mapped_blocks, + uint64_t trans_id, + uint64_t creation_time, + uint64_t snap_time) { + cur = 0; + bitmap = 0; + } + + void end_device() { + emit_bmp(true); + } + + void begin_named_mapping(string const &name) { } + + void end_named_mapping() { } + + void identifier(string const &name) { } + void range_map(uint64_t origin_begin, uint64_t, uint32_t, + uint64_t len) { + + uint64_t n = origin_begin / unit; + uint64_t i; + + assert(n >= cur); + assert(len > 0); + + /* + * Cover the gap between the last emitted unit and the current one. + */ + if (n > cur) + do { emit_bmp(); } while (cur < n); + + /* + * Emit partial unit. + */ + if (origin_begin & (unit - 1)) { + const uint64_t j = min(len, + (origin_begin & ~(unit - 1)) + unit - origin_begin); + for (i = origin_begin; i < origin_begin + j; i++) + bitmap |= 1ULL << (i & (unit - 1)); + if (j == len) + return; + + emit_bmp(); + + len -= j; + origin_begin = i; + } + + /* + * Emit full units until end. + */ + n = (origin_begin + len) / unit; + while (cur < n) { + bitmap = ~0; + emit_bmp(); + len -= unit; + } + origin_begin = cur * unit; + + /* + * Emit final unit. + */ + for (i = origin_begin; i < origin_begin + len; i++) + bitmap |= 1ULL << (i & (unit - 1)); + } + + void single_map(uint64_t origin_block, uint64_t, uint32_t) { + range_map(origin_block, 0, 0, 1); + } + + private: + ostream &out_; + + /** + * The entire virtual block allocation bitmap is segmented into 64-bit + * sub-bitmaps (units). + */ + uint64_t bitmap; + + /* + * Pointer to the current sub-bitmap (unit) that has not yet been + * emitted. + */ + uint64_t cur; + + /** + * Unit (sub-bitmap) size. Must be a power of 2. + */ + static const size_t unit = sizeof bitmap * CHAR_BIT; + + void emit_bmp(bool omit_if_zero = false) { + if (!bitmap && omit_if_zero) + out_.write((const char*)&bitmap, sizeof bitmap); + bitmap = 0; + cur++; + } + }; +} + +//---------------------------------------------------------------- + +extern "C" { + emitter::ptr create_emitter(ostream &out) { + return emitter::ptr(new binary_emitter(out)); + } +} + +//---------------------------------------------------------------- From 83dc84f790985b13e54864255b1e8693dbf3b8b6 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 24 Mar 2016 15:21:20 +0000 Subject: [PATCH 074/118] [contrib] ewheeler_emitter.so For thin_dump --- contrib/Makefile.in | 7 +++- contrib/ewheeler_emitter.cc | 84 +++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 contrib/ewheeler_emitter.cc diff --git a/contrib/Makefile.in b/contrib/Makefile.in index 58cfefc..1df15ec 100644 --- a/contrib/Makefile.in +++ b/contrib/Makefile.in @@ -1,6 +1,7 @@ PLUGINS=\ contrib/thin_sexp_emitter.so \ - contrib/tmakatos_emitter.so + contrib/tmakatos_emitter.so \ + contrib/ewheeler_emitter.so contrib: $(PLUGINS) @@ -12,4 +13,8 @@ contrib/tmakatos_emitter.so: contrib/tmakatos_emitter.o $(V)echo " [LD] $@" $(V)$(CC) -shared -Wl,-soname,tmakatos_emitter.so -o $@ $< +contrib/ewheeler_emitter.so: contrib/ewheeler_emitter.o + $(V)echo " [LD] $@" + $(V)$(CC) -shared -Wl,-soname,ewheeler_emitter.so -o $@ $< + diff --git a/contrib/ewheeler_emitter.cc b/contrib/ewheeler_emitter.cc new file mode 100644 index 0000000..fdc310d --- /dev/null +++ b/contrib/ewheeler_emitter.cc @@ -0,0 +1,84 @@ +#include "thin-provisioning/emitter.h" + +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + template + std::ostream &operator << (ostream &out, boost::optional const &maybe) { + if (maybe) + out << *maybe; + + return out; + } + + class old_emitter : public emitter { + public: + old_emitter(ostream &out) + : out_(out) { + } + + void begin_superblock(string const &uuid, + uint64_t time, + uint64_t trans_id, + boost::optional flags, + boost::optional version, + uint32_t data_block_size, + uint64_t nr_data_blocks, + boost::optional metadata_snap) { + data_block_size_ = data_block_size; + } + + void end_superblock() { + } + + void begin_device(uint32_t dev_id, + uint64_t mapped_blocks, + uint64_t trans_id, + uint64_t creation_time, + uint64_t snap_time) { + } + + void end_device() { + } + + void begin_named_mapping(string const &name) { + } + + void end_named_mapping() { + } + + void identifier(string const &name) { + } + + void range_map(uint64_t origin_begin, uint64_t data_begin, uint32_t time, uint64_t len) { + out_ << (data_block_size_ << 9)*origin_begin + << ":" << (data_block_size_ << 9)*len + << ":" << (data_block_size_ << 9)*data_begin + << endl; + } + + void single_map(uint64_t origin_block, uint64_t data_block, uint32_t time) { + out_ << (data_block_size_ << 9)*origin_block + << ":" << (data_block_size_ << 9) + << ":" << (data_block_size_ << 9)*data_block + << endl; + } + + private: + ostream &out_; + uint64_t data_block_size_; + }; +} + +//---------------------------------------------------------------- + +extern "C" { + emitter::ptr create_emitter(ostream &out) { + return emitter::ptr(new old_emitter(out)); + } +} + +//---------------------------------------------------------------- From 06e74656f66d421c14106f5dbe6789d92435fd1c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 24 Mar 2016 15:22:01 +0000 Subject: [PATCH 075/118] update ignore file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c53e066..4544b4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *~ *.o +*.so *.a *.gmo *_t From b8dbde1e64eb4f2d63c3504cb956be3066205720 Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Thu, 31 Mar 2016 13:00:36 +0300 Subject: [PATCH 076/118] use -ldl when linking dlopen etc. require -ldl to link --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 8af54a0..45ad359 100644 --- a/Makefile.in +++ b/Makefile.in @@ -137,7 +137,7 @@ CXXFLAGS+=@CXXDEBUG_FLAG@ CXXFLAGS+=@CXX_STRERROR_FLAG@ CXXFLAGS+=@LFS_FLAGS@ INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning -LIBS:=-laio -lexpat +LIBS:=-laio -lexpat -ldl ifeq ("@DEVTOOLS@", "yes") LIBS+=-lncurses From a5ed4258c04f8611817e78482f4e8e58d884e987 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 31 Mar 2016 16:14:11 +0100 Subject: [PATCH 077/118] [build] contrib target no longer depends on --enable-testing --- Makefile.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 8af54a0..d5b3a41 100644 --- a/Makefile.in +++ b/Makefile.in @@ -259,9 +259,10 @@ endif .PHONY: install +include contrib/Makefile + ifeq ("@TESTING@", "yes") include unit-tests/Makefile -include contrib/Makefile .PHONEY: features From 4a4dc1a5e0262e344f7e4f49aec779e9493ab543 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 31 Mar 2016 23:06:08 +0800 Subject: [PATCH 078/118] [btree_node_checker] first draft Spin-off from btree_damage_visitor --- Makefile.in | 1 + .../data-structures/btree_node_checker.cc | 127 +++++++++++ .../data-structures/btree_node_checker.h | 206 ++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 persistent-data/data-structures/btree_node_checker.cc create mode 100644 persistent-data/data-structures/btree_node_checker.h diff --git a/Makefile.in b/Makefile.in index 5e912d0..482f14f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -67,6 +67,7 @@ SOURCE=\ persistent-data/data-structures/bitset.cc \ persistent-data/data-structures/bloom_filter.cc \ persistent-data/data-structures/btree.cc \ + persistent-data/data-structures/btree_node_checker.cc \ persistent-data/error_set.cc \ persistent-data/file_utils.cc \ persistent-data/hex_dump.cc \ diff --git a/persistent-data/data-structures/btree_node_checker.cc b/persistent-data/data-structures/btree_node_checker.cc new file mode 100644 index 0000000..a6c77e0 --- /dev/null +++ b/persistent-data/data-structures/btree_node_checker.cc @@ -0,0 +1,127 @@ +#include "btree_node_checker.h" + +#include + +using persistent_data::btree_detail::btree_node_checker; + +//---------------------------------------------------------------- + +btree_node_checker::error_type btree_node_checker::get_last_error() { + return last_error_; +} + +std::string btree_node_checker::get_last_error_string() { + switch (last_error_) { + case BLOCK_NR_MISMATCH: + return block_nr_mismatch_string(); + case VALUE_SIZES_MISMATCH: + return value_sizes_mismatch_string(); + case MAX_ENTRIES_TOO_LARGE: + return max_entries_too_large_string(); + case MAX_ENTRIES_NOT_DIVISIBLE: + return max_entries_not_divisible_string(); + case NR_ENTRIES_TOO_LARGE: + return nr_entries_too_large_string(); + case NR_ENTRIES_TOO_SMALL: + return nr_entries_too_small_string(); + case KEYS_OUT_OF_ORDER: + return keys_out_of_order_string(); + case PARENT_KEY_MISMATCH: + return parent_key_mismatch_string(); + case LEAF_KEY_OVERLAPPED: + return leaf_key_overlapped_string(); + default: + return std::string(); + } +} + +void btree_node_checker::reset() { + last_error_ = NO_ERROR; +} + +std::string btree_node_checker::block_nr_mismatch_string() { + std::ostringstream out; + out << "block number mismatch: actually " + << error_location_ + << ", claims " << error_block_nr_; + + return out.str(); +} + +std::string btree_node_checker::value_sizes_mismatch_string() { + std::ostringstream out; + out << "value size mismatch: expected " << error_value_sizes_[1] + << ", but got " << error_value_sizes_[0] + << ". This is not the btree you are looking for." + << " (block " << error_location_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::max_entries_too_large_string() { + std::ostringstream out; + out << "max entries too large: " << error_max_entries_ + << " (block " << error_location_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::max_entries_not_divisible_string() { + std::ostringstream out; + out << "max entries is not divisible by 3: " << error_max_entries_ + << " (block " << error_location_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::nr_entries_too_large_string() { + std::ostringstream out; + out << "bad nr_entries: " + << error_nr_entries_ << " < " + << error_max_entries_ + << " (block " << error_location_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::nr_entries_too_small_string() { + std::ostringstream out; + out << "too few entries in btree_node: " + << error_nr_entries_ + << ", expected at least " + << (error_max_entries_ / 3) + << " (block " << error_location_ + << ", max_entries = " << error_max_entries_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::keys_out_of_order_string() { + std::ostringstream out; + out << "keys are out of order, " + << error_keys_[0] << " <= " << error_keys_[1] + << " (block " << error_location_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::parent_key_mismatch_string() { + std::ostringstream out; + out << "parent key mismatch: parent was " << error_keys_[1] + << ", but lowest in node was " << error_keys_[0] + << " (block " << error_location_ << ")"; + + return out.str(); +} + +std::string btree_node_checker::leaf_key_overlapped_string() { + std::ostringstream out; + out << "the last key of the previous leaf was " << error_keys_[1] + << " and the first key of this leaf is " << error_keys_[0] + << " (block " << error_location_ << ")"; + + return out.str(); +} + +//---------------------------------------------------------------- + diff --git a/persistent-data/data-structures/btree_node_checker.h b/persistent-data/data-structures/btree_node_checker.h new file mode 100644 index 0000000..926f613 --- /dev/null +++ b/persistent-data/data-structures/btree_node_checker.h @@ -0,0 +1,206 @@ +#ifndef BTREE_NODE_CHECKER_H +#define BTREE_NODE_CHECKER_H + +#include "block-cache/block_cache.h" +#include "persistent-data/block.h" +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_disk_structures.h" + +#include +#include + +using bcache::block_address; + +//---------------------------------------------------------------- + +namespace persistent_data { + namespace btree_detail { + class btree_node_checker { + public: + enum error_type { + NO_ERROR, + BLOCK_NR_MISMATCH, + VALUE_SIZES_MISMATCH, + MAX_ENTRIES_TOO_LARGE, + MAX_ENTRIES_NOT_DIVISIBLE, + NR_ENTRIES_TOO_LARGE, + NR_ENTRIES_TOO_SMALL, + KEYS_OUT_OF_ORDER, + VALUE_SIZE_MISMATCH, + PARENT_KEY_MISMATCH, + LEAF_KEY_OVERLAPPED, + }; + + btree_node_checker(): + last_error_(NO_ERROR), + error_location_(0), + error_block_nr_(0), + error_nr_entries_(0), + error_max_entries_(0), + error_value_sizes_{0, 0}, + error_keys_{0, 0} { + } + + template + bool check_block_nr(btree_detail::node_ref const &n) { + if (n.get_location() != n.get_block_nr()) { + last_error_ = BLOCK_NR_MISMATCH; + error_block_nr_ = n.get_block_nr(); + error_location_ = n.get_location(); + + return false; + } + + return true; + } + + template + bool check_value_size(btree_detail::node_ref const &n) { + if (!n.value_sizes_match()) { + last_error_ = VALUE_SIZES_MISMATCH; + error_location_ = n.get_location(); + error_value_sizes_[0] = n.get_value_size(); + error_value_sizes_[1] = sizeof(typename ValueTraits::disk_type); + return false; + } + + return true; + } + + template + bool check_max_entries(btree_detail::node_ref const &n) { + size_t elt_size = sizeof(uint64_t) + n.get_value_size(); + if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { + last_error_ = MAX_ENTRIES_TOO_LARGE; + error_location_ = n.get_location(); + error_max_entries_ = n.get_max_entries(); + + return false; + } + + if (n.get_max_entries() % 3) { + last_error_ = MAX_ENTRIES_NOT_DIVISIBLE; + error_location_ = n.get_location(); + error_max_entries_ = n.get_max_entries(); + + return false; + } + + return true; + } + + template + bool check_nr_entries(btree_detail::node_ref const &n, + bool is_root) { + if (n.get_nr_entries() > n.get_max_entries()) { + last_error_ = NR_ENTRIES_TOO_LARGE; + error_location_ = n.get_location(); + error_nr_entries_ = n.get_nr_entries(); + error_max_entries_ = n.get_max_entries(); + + return false; + } + + block_address min = n.get_max_entries() / 3; + if (!is_root && (n.get_nr_entries() < min)) { + last_error_ = NR_ENTRIES_TOO_SMALL; + error_location_ = n.get_location(); + error_nr_entries_ = n.get_nr_entries(); + error_max_entries_ = n.get_max_entries(); + + return false; + } + + return true; + } + + template + bool check_ordered_keys(btree_detail::node_ref const &n) { + unsigned nr_entries = n.get_nr_entries(); + + if (nr_entries == 0) + return true; // can only happen if a root node + + uint64_t last_key = n.key_at(0); + + for (unsigned i = 1; i < nr_entries; i++) { + uint64_t k = n.key_at(i); + if (k <= last_key) { + last_error_ = KEYS_OUT_OF_ORDER; + error_location_ = n.get_location(); + error_keys_[0] = k; + error_keys_[1] = last_key; + + return false; + } + last_key = k; + } + + return true; + } + + template + bool check_parent_key(btree_detail::node_ref const &n, + boost::optional key) { + if (!key) + return true; + + if (*key > n.key_at(0)) { + last_error_ = PARENT_KEY_MISMATCH; + error_location_ = n.get_location(); + error_keys_[0] = n.key_at(0); + error_keys_[1] = *key; + + return false; + } + + return true; + } + + template + bool check_leaf_key(btree_detail::node_ref const &n, + boost::optional key) { + if (n.get_nr_entries() == 0) + return true; // can only happen if a root node + + if (key && *key >= n.key_at(0)) { + last_error_ = LEAF_KEY_OVERLAPPED; + error_location_ = n.get_location(); + error_keys_[0] = n.key_at(0); + error_keys_[1] = *key; + + return false; + } + + return true; + } + + error_type get_last_error(); + std::string get_last_error_string(); + void reset(); + + private: + std::string block_nr_mismatch_string(); + std::string value_sizes_mismatch_string(); + std::string max_entries_too_large_string(); + std::string max_entries_not_divisible_string(); + std::string nr_entries_too_large_string(); + std::string nr_entries_too_small_string(); + std::string keys_out_of_order_string(); + std::string parent_key_mismatch_string(); + std::string leaf_key_overlapped_string(); + + error_type last_error_; + block_address error_location_; + block_address error_block_nr_; + uint32_t error_nr_entries_; + uint32_t error_max_entries_; + uint32_t error_value_sizes_[2]; + uint64_t error_keys_[2]; + }; + } +} + +//---------------------------------------------------------------- + +#endif From 6dc9a90fecb3aeff2fa075859d036d9a966e71ea Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 31 Mar 2016 23:08:14 +0800 Subject: [PATCH 079/118] [counting_visitor] fix unnecessary value visiting 1. Do not inherit btree_damage_visitor to avoid unnecessary value visiting. (reverts commit b22495997a8bf427741225ef86edf267ee746d7c) 2. Use btree_node_checker to do node checking --- .../data-structures/btree_counter.h | 92 ++++++++++++------- .../data-structures/btree_damage_visitor.h | 5 +- 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index 6e52c29..1b8c7a2 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -2,44 +2,36 @@ #define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H #include "persistent-data/data-structures/btree.h" -#include "persistent-data/data-structures/btree_base_visitor.h" -#include "persistent-data/data-structures/btree_damage_visitor.h" +#include "persistent-data/data-structures/btree_node_checker.h" #include "persistent-data/block_counter.h" //---------------------------------------------------------------- namespace persistent_data { namespace btree_count_detail { - template - class counting_visitor : public btree_damage_visitor { - typedef btree_damage_visitor BtreeDamageVisitor; + template + class counting_visitor : public btree::visitor { public: typedef btree tree; - counting_visitor(ValueVisitor &value_visitor, - DamageVisitor &damage_visitor, - block_counter &bc, - ValueCounter &vc) - : BtreeDamageVisitor(value_visitor, damage_visitor, false), - bc_(bc), + counting_visitor(block_counter &bc, ValueCounter &vc) + : bc_(bc), vc_(vc) { } virtual bool visit_internal(node_location const &l, typename tree::internal_node const &n) { - return BtreeDamageVisitor::visit_internal(l, n) ? - visit_node(n) : false; + return check_internal(l, n) ? visit_node(n) : false; } virtual bool visit_internal_leaf(node_location const &l, typename tree::internal_node const &n) { - return BtreeDamageVisitor::visit_internal_leaf(l, n) ? - visit_node(n) : false; + return check_leaf(l, n) ? visit_node(n) : false; } virtual bool visit_leaf(node_location const &l, typename tree::leaf_node const &n) { - if (BtreeDamageVisitor::visit_leaf(l, n) && visit_node(n)) { + if (check_leaf(l, n) && visit_node(n)) { unsigned nr = n.get_nr_entries(); for (unsigned i = 0; i < nr; i++) { @@ -55,7 +47,57 @@ namespace persistent_data { return false; } + typedef typename btree::visitor::error_outcome error_outcome; + + error_outcome error_accessing_node(node_location const &l, block_address b, + std::string const &what) { + return btree::visitor::EXCEPTION_HANDLED; + } + private: + bool check_internal(node_location const &l, + btree_detail::node_ref const &n) { + if (!checker_.check_block_nr(n) || + !checker_.check_value_size(n) || + !checker_.check_max_entries(n) || + !checker_.check_nr_entries(n, l.is_sub_root()) || + !checker_.check_ordered_keys(n) || + !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional() : l.key)) + return false; + + if (l.is_sub_root()) + new_root(l.level()); + + return true; + } + + template + bool check_leaf(node_location const &l, + btree_detail::node_ref const &n) { + if (!checker_.check_block_nr(n) || + !checker_.check_value_size(n) || + !checker_.check_max_entries(n) || + !checker_.check_nr_entries(n, l.is_sub_root()) || + !checker_.check_ordered_keys(n) || + !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional() : l.key)) + return false; + + if (l.is_sub_root()) + new_root(l.level()); + + bool r = checker_.check_leaf_key(n, last_leaf_key_[l.level()]); + if (r && n.get_nr_entries() > 0) + last_leaf_key_[l.level()] = n.key_at(n.get_nr_entries() - 1); + + return r; + } + + void new_root(unsigned level) { + // we're starting a new subtree, so should + // reset the last_leaf value. + last_leaf_key_[level] = boost::optional(); + } + template bool visit_node(Node const &n) { block_address b = n.get_location(); @@ -66,6 +108,8 @@ namespace persistent_data { block_counter &bc_; ValueCounter &vc_; + btree_node_checker checker_; + boost::optional last_leaf_key_[Levels]; }; } @@ -94,21 +138,7 @@ namespace persistent_data { // is not corrupt. template void count_btree_blocks(btree const &tree, block_counter &bc, ValueCounter &vc) { - typedef noop_value_visitor NoopValueVisitor; - NoopValueVisitor noop_vv; - noop_damage_visitor noop_dv; - btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); - tree.visit_depth_first(v); - } - - template - void count_btree_blocks(btree const &tree, block_counter &bc) { - typedef noop_value_visitor NoopValueVisitor; - NoopValueVisitor noop_vv; - noop_damage_visitor noop_dv; - typedef noop_value_counter NoopValueCounter; - NoopValueCounter vc; - btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); + btree_count_detail::counting_visitor v(bc, vc); tree.visit_depth_first(v); } } diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 8be623b..5a96d5d 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -158,9 +158,8 @@ namespace persistent_data { typedef boost::optional maybe_run64; btree_damage_visitor(ValueVisitor &value_visitor, - DamageVisitor &damage_visitor, - bool avoid_repeated_visits = true) - : avoid_repeated_visits_(avoid_repeated_visits), + DamageVisitor &damage_visitor) + : avoid_repeated_visits_(true), value_visitor_(value_visitor), damage_visitor_(damage_visitor) { } From c6c508606857e4c6db70f5794063a46255360d53 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 2 Apr 2016 16:14:24 +0800 Subject: [PATCH 080/118] [thin_ll_dump] cleanup: use btree_node_checker --- thin-provisioning/thin_ll_dump.cc | 72 ++++--------------------------- 1 file changed, 8 insertions(+), 64 deletions(-) diff --git a/thin-provisioning/thin_ll_dump.cc b/thin-provisioning/thin_ll_dump.cc index 2a20237..f16a745 100644 --- a/thin-provisioning/thin_ll_dump.cc +++ b/thin-provisioning/thin_ll_dump.cc @@ -24,6 +24,8 @@ #include "persistent-data/file_utils.h" #include "persistent-data/data-structures/btree.h" #include "persistent-data/data-structures/btree_counter.h" +#include "persistent-data/data-structures/btree_damage_visitor.h" +#include "persistent-data/data-structures/btree_node_checker.h" #include "persistent-data/data-structures/simple_traits.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/space-maps/disk_structures.h" @@ -33,70 +35,11 @@ #include "version.h" using namespace thin_provisioning; +using namespace persistent_data; //---------------------------------------------------------------- namespace { - // extracted from btree_damage_visitor.h - template - bool check_block_nr(node const &n) { - if (n.get_location() != n.get_block_nr()) { - return false; - } - return true; - } - - // extracted from btree_damage_visitor.h - template - bool check_max_entries(node const &n) { - size_t elt_size = sizeof(uint64_t) + n.get_value_size(); - if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { - return false; - } - - if (n.get_max_entries() % 3) { - return false; - } - - return true; - } - - // extracted from btree_damage_visitor.h - template - bool check_nr_entries(node const &n, bool is_root) { - if (n.get_nr_entries() > n.get_max_entries()) { - return false; - } - - block_address min = n.get_max_entries() / 3; - if (!is_root && (n.get_nr_entries() < min)) { - return false; - } - - return true; - } - - // extracted from btree_damage_visitor.h - template - bool check_ordered_keys(node const &n) { - unsigned nr_entries = n.get_nr_entries(); - - if (nr_entries == 0) - return true; // can only happen if a root node - - uint64_t last_key = n.key_at(0); - - for (unsigned i = 1; i < nr_entries; i++) { - uint64_t k = n.key_at(i); - if (k <= last_key) { - return false; - } - last_key = k; - } - - return true; - } - transaction_manager::ptr open_tm(block_manager<>::ptr bm) { space_map::ptr sm(new core_map(bm->get_nr_blocks())); @@ -136,19 +79,20 @@ namespace { if ((n.get_value_size() == sizeof(mapping_tree_detail::block_traits::disk_type) || n.get_value_size() == sizeof(device_tree_detail::device_details_traits::disk_type)) && !bc_.get_count(n.get_location()) && - check_block_nr(n) && + checker_.check_block_nr(n) && (((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) || (flags & LEAF_NODE)) && nv_->check_raw(n.raw()) && - check_max_entries(n) && - check_nr_entries(n, true) && - check_ordered_keys(n)) + checker_.check_max_entries(n) && + checker_.check_nr_entries(n, true) && + checker_.check_ordered_keys(n)) return true; return false; } bcache::validator::ptr nv_; block_counter const &bc_; + btree_detail::btree_node_checker checker_; }; //------------------------------------------------------------------- From 1dce79bd5551fef6cac310b2a8e7a69923b7160e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 2 Apr 2016 17:15:00 +0800 Subject: [PATCH 081/118] [btree_damage_visitor] cleanup: use btree_node_checker --- .../data-structures/btree_damage_visitor.h | 201 ++++-------------- 1 file changed, 40 insertions(+), 161 deletions(-) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 5a96d5d..94e255b 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -2,6 +2,7 @@ #define PERSISTENT_DATA_DATA_STRUCTURES_DAMAGE_VISITOR_H #include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_node_checker.h" #include "persistent-data/run.h" //---------------------------------------------------------------- @@ -222,44 +223,55 @@ namespace persistent_data { bool check_internal(node_location const &loc, btree_detail::node_ref const &n) { - if (!already_visited(n) && - check_block_nr(n) && - check_value_size(n) && - check_max_entries(n) && - check_nr_entries(n, loc.is_sub_root()) && - check_ordered_keys(n) && - check_parent_key(loc.is_sub_root() ? boost::optional() : loc.key, n)) { - if (loc.is_sub_root()) - new_root(loc.level()); + if (already_visited(n)) + return false; + else if (!checker_.check_block_nr(n) || + !checker_.check_value_size(n) || + !checker_.check_max_entries(n) || + !checker_.check_nr_entries(n, loc.is_sub_root()) || + !checker_.check_ordered_keys(n) || + !checker_.check_parent_key(n, loc.is_sub_root() ? boost::optional() : loc.key)) { + report_damage(checker_.get_last_error_string()); - good_internal(n.key_at(0)); - return true; + return false; } - return false; + if (loc.is_sub_root()) + new_root(loc.level()); + + good_internal(n.key_at(0)); + + return true; } template bool check_leaf(node_location const &loc, btree_detail::node_ref const &n) { - if (!already_visited(n) && - check_block_nr(n) && - check_value_size(n) && - check_max_entries(n) && - check_nr_entries(n, loc.is_sub_root()) && - check_ordered_keys(n) && - check_parent_key(loc.is_sub_root() ? boost::optional() : loc.key, n)) { - if (loc.is_sub_root()) - new_root(loc.level()); + if (already_visited(n)) + return false; + else if (!checker_.check_block_nr(n) || + !checker_.check_value_size(n) || + !checker_.check_max_entries(n) || + !checker_.check_nr_entries(n, loc.is_sub_root()) || + !checker_.check_ordered_keys(n) || + !checker_.check_parent_key(n, loc.is_sub_root() ? boost::optional() : loc.key)) { + report_damage(checker_.get_last_error_string()); - bool r = check_leaf_key(loc.level(), n); - if (r && n.get_nr_entries() > 0) - good_leaf(n.key_at(0), n.key_at(n.get_nr_entries() - 1) + 1); - - return r; + return false; } - return false; + if (loc.is_sub_root()) + new_root(loc.level()); + + bool r = checker_.check_leaf_key(n, last_leaf_key_[loc.level()]); + if (!r) + report_damage(checker_.get_last_error_string()); + else if (n.get_nr_entries() > 0) { + last_leaf_key_[loc.level()] = n.key_at(n.get_nr_entries() - 1); + good_leaf(n.key_at(0), n.key_at(n.get_nr_entries() - 1) + 1); + } + + return r; } template @@ -276,140 +288,6 @@ namespace persistent_data { return false; } - template - bool check_block_nr(node const &n) { - if (n.get_location() != n.get_block_nr()) { - std::ostringstream out; - out << "block number mismatch: actually " - << n.get_location() - << ", claims " << n.get_block_nr(); - - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_value_size(node const &n) { - if (!n.value_sizes_match()) { - report_damage(n.value_mismatch_string()); - return false; - } - - return true; - } - - template - bool check_max_entries(node const &n) { - size_t elt_size = sizeof(uint64_t) + n.get_value_size(); - if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { - std::ostringstream out; - out << "max entries too large: " << n.get_max_entries() - << " (block " << n.get_location() << ")"; - report_damage(out.str()); - return false; - } - - if (n.get_max_entries() % 3) { - std::ostringstream out; - out << "max entries is not divisible by 3: " << n.get_max_entries() - << " (block " << n.get_location() << ")"; - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_nr_entries(node const &n, bool is_root) { - if (n.get_nr_entries() > n.get_max_entries()) { - std::ostringstream out; - out << "bad nr_entries: " - << n.get_nr_entries() << " < " - << n.get_max_entries() - << " (block " << n.get_location() << ")"; - report_damage(out.str()); - return false; - } - - block_address min = n.get_max_entries() / 3; - if (!is_root && (n.get_nr_entries() < min)) { - ostringstream out; - out << "too few entries in btree_node: " - << n.get_nr_entries() - << ", expected at least " - << min - << " (block " << n.get_location() - << ", max_entries = " << n.get_max_entries() << ")"; - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_ordered_keys(node const &n) { - unsigned nr_entries = n.get_nr_entries(); - - if (nr_entries == 0) - return true; // can only happen if a root node - - uint64_t last_key = n.key_at(0); - - for (unsigned i = 1; i < nr_entries; i++) { - uint64_t k = n.key_at(i); - if (k <= last_key) { - ostringstream out; - out << "keys are out of order, " << k << " <= " << last_key - << " (block " << n.get_location() << ")"; - report_damage(out.str()); - return false; - } - last_key = k; - } - - return true; - } - - template - bool check_parent_key(boost::optional key, node const &n) { - if (!key) - return true; - - if (*key > n.key_at(0)) { - ostringstream out; - out << "parent key mismatch: parent was " << *key - << ", but lowest in node was " << n.key_at(0) - << " (block " << n.get_location() << ")"; - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_leaf_key(unsigned level, node const &n) { - if (n.get_nr_entries() == 0) - return true; // can only happen if a root node - - if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) { - ostringstream out; - out << "the last key of the previous leaf was " << *last_leaf_key_[level] - << " and the first key of this leaf is " << n.key_at(0) - << " (block " << n.get_location() << ")"; - report_damage(out.str()); - return false; - } - - last_leaf_key_[level] = n.key_at(n.get_nr_entries() - 1); - return true; - } - void new_root(unsigned level) { // we're starting a new subtree, so should // reset the last_leaf value. @@ -487,6 +365,7 @@ namespace persistent_data { std::set seen_; boost::optional last_leaf_key_[Levels]; + btree_node_checker checker_; path_tracker path_tracker_; damage_tracker dt_; std::list damage_reasons_; From c8aabf2948d39baa52cb7e7d83f31dbf38baea34 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 2 Apr 2016 23:15:26 +0800 Subject: [PATCH 082/118] [metadata_counter] fix repeated counting of trees --- thin-provisioning/metadata_counter.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/thin-provisioning/metadata_counter.cc b/thin-provisioning/metadata_counter.cc index bf3f809..e36fde6 100644 --- a/thin-provisioning/metadata_counter.cc +++ b/thin-provisioning/metadata_counter.cc @@ -66,7 +66,6 @@ void thin_provisioning::count_metadata(transaction_manager::ptr tm, count_trees(tm, snap, bc); } - count_trees(tm, sb, bc); count_space_maps(tm, sb, bc); } From f20e2a0f401c1e30c65ff29e76c2af97eb01daf9 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 2 Apr 2016 23:24:24 +0800 Subject: [PATCH 083/118] [thin_check] cleanup: use metadata_counter --- thin-provisioning/thin_check.cc | 48 ++------------------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/thin-provisioning/thin_check.cc b/thin-provisioning/thin_check.cc index 2451db2..f2c57cb 100644 --- a/thin-provisioning/thin_check.cc +++ b/thin-provisioning/thin_check.cc @@ -31,6 +31,7 @@ #include "persistent-data/file_utils.h" #include "thin-provisioning/device_tree.h" #include "thin-provisioning/mapping_tree.h" +#include "thin-provisioning/metadata_counter.h" #include "thin-provisioning/superblock.h" #include "thin-provisioning/commands.h" @@ -169,58 +170,13 @@ namespace { bool clear_needs_check_flag_on_success; }; - void count_trees(transaction_manager::ptr tm, - superblock_detail::superblock &sb, - block_counter &bc) { - - // Count the device tree - { - noop_value_counter vc; - device_tree dtree(*tm, sb.device_details_root_, - device_tree_detail::device_details_traits::ref_counter()); - count_btree_blocks(dtree, bc, vc); - } - - // Count the mapping tree - { - noop_value_counter vc; - mapping_tree mtree(*tm, sb.data_mapping_root_, - mapping_tree_detail::block_traits::ref_counter(tm->get_sm())); - count_btree_blocks(mtree, bc, vc); - } - } - error_state check_space_map_counts(flags const &fs, nested_output &out, superblock_detail::superblock &sb, block_manager<>::ptr bm, transaction_manager::ptr tm) { block_counter bc; - // Count the superblock - bc.inc(superblock_detail::SUPERBLOCK_LOCATION); - count_trees(tm, sb, bc); - - // Count the metadata snap, if present - if (sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) { - bc.inc(sb.metadata_snap_); - - superblock_detail::superblock snap = read_superblock(bm, sb.metadata_snap_); - count_trees(tm, snap, bc); - } - - // Count the metadata space map - { - persistent_space_map::ptr metadata_sm = - open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); - metadata_sm->count_metadata(bc); - } - - // Count the data space map - { - persistent_space_map::ptr data_sm = - open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); - data_sm->count_metadata(bc); - } + count_metadata(tm, sb, bc); // Finally we need to check the metadata space map agrees // with the counts we've just calculated. From 9322fc9f142ea97e50fea435b1bff91631b5f9d2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 5 Apr 2016 15:41:42 +0800 Subject: [PATCH 084/118] [btree_damage_visitor] cleanup: remove redundant statements --- .../data-structures/btree_damage_visitor.h | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 94e255b..1ff16a8 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -223,6 +223,9 @@ namespace persistent_data { bool check_internal(node_location const &loc, btree_detail::node_ref const &n) { + if (loc.is_sub_root()) + new_root(loc.level()); + if (already_visited(n)) return false; else if (!checker_.check_block_nr(n) || @@ -236,9 +239,6 @@ namespace persistent_data { return false; } - if (loc.is_sub_root()) - new_root(loc.level()); - good_internal(n.key_at(0)); return true; @@ -247,6 +247,9 @@ namespace persistent_data { template bool check_leaf(node_location const &loc, btree_detail::node_ref const &n) { + if (loc.is_sub_root()) + new_root(loc.level()); + if (already_visited(n)) return false; else if (!checker_.check_block_nr(n) || @@ -254,24 +257,19 @@ namespace persistent_data { !checker_.check_max_entries(n) || !checker_.check_nr_entries(n, loc.is_sub_root()) || !checker_.check_ordered_keys(n) || - !checker_.check_parent_key(n, loc.is_sub_root() ? boost::optional() : loc.key)) { + !checker_.check_parent_key(n, loc.is_sub_root() ? boost::optional() : loc.key) || + !checker_.check_leaf_key(n, last_leaf_key_[loc.level()])) { report_damage(checker_.get_last_error_string()); return false; } - if (loc.is_sub_root()) - new_root(loc.level()); - - bool r = checker_.check_leaf_key(n, last_leaf_key_[loc.level()]); - if (!r) - report_damage(checker_.get_last_error_string()); - else if (n.get_nr_entries() > 0) { + if (n.get_nr_entries() > 0) { last_leaf_key_[loc.level()] = n.key_at(n.get_nr_entries() - 1); good_leaf(n.key_at(0), n.key_at(n.get_nr_entries() - 1) + 1); } - return r; + return true; } template From 810e86e675170c04bbf33e0cc3a8809ae2893ca0 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 5 Apr 2016 17:05:28 +0800 Subject: [PATCH 085/118] [counting_visitor] cleanup: remove redundant statements --- .../data-structures/btree_counter.h | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index 1b8c7a2..ec74e86 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -57,6 +57,9 @@ namespace persistent_data { private: bool check_internal(node_location const &l, btree_detail::node_ref const &n) { + if (l.is_sub_root()) + new_root(l.level()); + if (!checker_.check_block_nr(n) || !checker_.check_value_size(n) || !checker_.check_max_entries(n) || @@ -65,31 +68,28 @@ namespace persistent_data { !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional() : l.key)) return false; - if (l.is_sub_root()) - new_root(l.level()); - return true; } template bool check_leaf(node_location const &l, btree_detail::node_ref const &n) { + if (l.is_sub_root()) + new_root(l.level()); + if (!checker_.check_block_nr(n) || !checker_.check_value_size(n) || !checker_.check_max_entries(n) || !checker_.check_nr_entries(n, l.is_sub_root()) || !checker_.check_ordered_keys(n) || - !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional() : l.key)) + !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional() : l.key) || + !checker_.check_leaf_key(n, last_leaf_key_[l.level()])) return false; - if (l.is_sub_root()) - new_root(l.level()); - - bool r = checker_.check_leaf_key(n, last_leaf_key_[l.level()]); - if (r && n.get_nr_entries() > 0) + if (n.get_nr_entries() > 0) last_leaf_key_[l.level()] = n.key_at(n.get_nr_entries() - 1); - return r; + return true; } void new_root(unsigned level) { From ab958ee752d7faa75967b3d631cf18591852a4f0 Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Thu, 31 Mar 2016 14:20:12 +0300 Subject: [PATCH 086/118] improve error message when failing to stat block device --- persistent-data/file_utils.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/persistent-data/file_utils.cc b/persistent-data/file_utils.cc index 88ee945..4853040 100644 --- a/persistent-data/file_utils.cc +++ b/persistent-data/file_utils.cc @@ -21,7 +21,8 @@ persistent_data::get_nr_blocks(string const &path, sector_t block_size) int r = ::stat(path.c_str(), &info); if (r) - throw runtime_error("Couldn't stat dev path"); + throw runtime_error("Couldn't stat dev path " + path + ": " + + strerror(errno)); if (S_ISREG(info.st_mode) && info.st_size) nr_blocks = div_up(info.st_size, block_size); From 3464f005ff2c544541f51a9f34a1a551bf4f06ac Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Wed, 9 Mar 2016 13:26:13 +0300 Subject: [PATCH 087/118] add configure option to enable static linking (cherry picked from commit c6a2620f5de7aeea4e9c20ff4e6d1befb4b2e935) Conflicts: configure.ac --- Makefile.in | 4 ++++ configure.ac | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/Makefile.in b/Makefile.in index 482f14f..4373086 100644 --- a/Makefile.in +++ b/Makefile.in @@ -150,6 +150,10 @@ else CXXLIB+=-lstdc++ endif +ifeq ("@STATIC@", "yes") +LDFLAGS+=-static +endif + INSTALL:=@INSTALL@ PREFIX:=@prefix@ BINDIR:=$(DESTDIR)$(PREFIX)/sbin diff --git a/configure.ac b/configure.ac index 9b0b2ce..efca586 100644 --- a/configure.ac +++ b/configure.ac @@ -153,6 +153,14 @@ AC_ARG_ENABLE(static_cxx, STATIC_CXX=$enableval, STATIC_CXX=no) AC_MSG_RESULT($STATIC_CXX) +################################################################################ +dnl -- Enable static linking. +AC_MSG_CHECKING(whether to statically link) +AC_ARG_ENABLE(static, + AC_HELP_STRING(--enable-static, [enable static link]), + STATIC=$enableval, STATIC=no) +AC_MSG_RESULT($STATIC) + ################################################################################ dnl -- Check for getopt AC_CHECK_HEADERS(getopt.h, AC_DEFINE([HAVE_GETOPTLONG], 1, [Define to 1 if getopt_long is available.])) @@ -182,6 +190,7 @@ AC_SUBST(TESTING) AC_SUBST(THIN_PROVISIONING_TOOLS_VERSION) AC_SUBST(STATIC_CXX) AC_SUBST(DEVTOOLS) +AC_SUBST(STATIC) ################################################################################ dnl -- First and last lines should not contain files to generate in order to From 088d5ac05eec54940d07226d57f771ed0d0a687b Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Wed, 6 Apr 2016 14:06:29 +0300 Subject: [PATCH 088/118] compile 3rd party emitters into static lib before creating shared lib This patch adds an additional step in the compilation of 3rd party emitters: we first create the static library and then create the shared library. This is necessary in order to build the emitters into the static binary, which is implemented in a subsequent patch. --- contrib/Makefile.in | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/contrib/Makefile.in b/contrib/Makefile.in index 1df15ec..0b6c28f 100644 --- a/contrib/Makefile.in +++ b/contrib/Makefile.in @@ -1,20 +1,24 @@ +PLUGIN_LIBS= \ + contrib/thin_sexp_emitter.a \ + contrib/tmakatos_emitter.a \ + contrib/ewheeler_emitter.a + PLUGINS=\ contrib/thin_sexp_emitter.so \ contrib/tmakatos_emitter.so \ contrib/ewheeler_emitter.so -contrib: $(PLUGINS) +contrib: $(PLUGINS) $(PLUGIN_LIBS) -contrib/thin_sexp_emitter.so: contrib/thin_sexp_emitter.o +contrib/%.o: contrib/%.cc + $(V)echo " [CC] $@" + $(V)$(CC) $^ -o $@ + +contrib/%.a: contrib/%.o + $(V)echo " [AR] $@" + $(V)ar rcs $@ $^ + +contrib/%.so: contrib/%.a $(V)echo " [LD] $@" - $(V)$(CC) -shared -Wl,-soname,thin_sexp_emitter.so -o $@ $< - -contrib/tmakatos_emitter.so: contrib/tmakatos_emitter.o - $(V)echo " [LD] $@" - $(V)$(CC) -shared -Wl,-soname,tmakatos_emitter.so -o $@ $< - -contrib/ewheeler_emitter.so: contrib/ewheeler_emitter.o - $(V)echo " [LD] $@" - $(V)$(CC) -shared -Wl,-soname,ewheeler_emitter.so -o $@ $< - + $(V)$(CC) -shared -Wl,-soname,$@ -o $@ $< From 35cfc3b90a8ece7e7d99b96f9eddf2fe428c255e Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Wed, 6 Apr 2016 14:13:21 +0300 Subject: [PATCH 089/118] introduce header file for the tmakatos (a binary) emitter This is necessary because the emitter's symbols need to be resolved at compile time. Only the tamakatos emitter header file is introduced, as it is trivial to do so for the other emitters. --- contrib/tmakatos_emitter.cc | 184 ++++++++++++++++-------------------- contrib/tmakatos_emitter.h | 69 ++++++++++++++ 2 files changed, 150 insertions(+), 103 deletions(-) create mode 100644 contrib/tmakatos_emitter.h diff --git a/contrib/tmakatos_emitter.cc b/contrib/tmakatos_emitter.cc index 4699077..32ad397 100644 --- a/contrib/tmakatos_emitter.cc +++ b/contrib/tmakatos_emitter.cc @@ -1,4 +1,5 @@ #include "thin-provisioning/emitter.h" +#include "contrib/tmakatos_emitter.h" #include #include @@ -8,7 +9,7 @@ using namespace thin_provisioning; //---------------------------------------------------------------- -namespace { +namespace tmakatos_emitter { template std::ostream &operator << (ostream &out, boost::optional const &maybe) { if (maybe) @@ -20,131 +21,108 @@ namespace { //------------------------------------------------ // binary generator //------------------------------------------------ - class binary_emitter : public emitter { - public: - binary_emitter(ostream &out) - : out_(out) { - } + binary_emitter::binary_emitter(ostream &out): out_(out) { + } - void begin_superblock(string const &uuid, - uint64_t time, - uint64_t trans_id, - boost::optional flags, - boost::optional version, - uint32_t data_block_size, - uint64_t nr_data_blocks, - boost::optional metadata_snap) { - } + void binary_emitter::begin_superblock(string const &uuid, + uint64_t time, + uint64_t trans_id, + boost::optional flags, + boost::optional version, + uint32_t data_block_size, + uint64_t nr_data_blocks, + boost::optional metadata_snap) { + } - void end_superblock() { - } + void binary_emitter::end_superblock() { + } - void begin_device(uint32_t dev_id, - uint64_t mapped_blocks, - uint64_t trans_id, - uint64_t creation_time, - uint64_t snap_time) { - cur = 0; - bitmap = 0; - } + void binary_emitter::begin_device(uint32_t dev_id, + uint64_t mapped_blocks, + uint64_t trans_id, + uint64_t creation_time, + uint64_t snap_time) { + cur = 0; + bitmap = 0; + } - void end_device() { - emit_bmp(true); - } + void binary_emitter::end_device() { + emit_bmp(true); + } - void begin_named_mapping(string const &name) { } + void binary_emitter::begin_named_mapping(string const &name) { } - void end_named_mapping() { } + void binary_emitter::end_named_mapping() { } - void identifier(string const &name) { } - void range_map(uint64_t origin_begin, uint64_t, uint32_t, - uint64_t len) { + void binary_emitter::identifier(string const &name) { } - uint64_t n = origin_begin / unit; - uint64_t i; + void binary_emitter::range_map(uint64_t origin_begin, uint64_t, uint32_t, + uint64_t len) { - assert(n >= cur); - assert(len > 0); + uint64_t n = origin_begin / unit; + uint64_t i; - /* - * Cover the gap between the last emitted unit and the current one. - */ - if (n > cur) - do { emit_bmp(); } while (cur < n); - - /* - * Emit partial unit. - */ - if (origin_begin & (unit - 1)) { - const uint64_t j = min(len, - (origin_begin & ~(unit - 1)) + unit - origin_begin); - for (i = origin_begin; i < origin_begin + j; i++) - bitmap |= 1ULL << (i & (unit - 1)); - if (j == len) - return; - - emit_bmp(); - - len -= j; - origin_begin = i; - } - - /* - * Emit full units until end. - */ - n = (origin_begin + len) / unit; - while (cur < n) { - bitmap = ~0; - emit_bmp(); - len -= unit; - } - origin_begin = cur * unit; - - /* - * Emit final unit. - */ - for (i = origin_begin; i < origin_begin + len; i++) - bitmap |= 1ULL << (i & (unit - 1)); - } - - void single_map(uint64_t origin_block, uint64_t, uint32_t) { - range_map(origin_block, 0, 0, 1); - } - - private: - ostream &out_; - - /** - * The entire virtual block allocation bitmap is segmented into 64-bit - * sub-bitmaps (units). - */ - uint64_t bitmap; + assert(n >= cur); + assert(len > 0); /* - * Pointer to the current sub-bitmap (unit) that has not yet been - * emitted. + * Cover the gap between the last emitted unit and the current one. */ - uint64_t cur; + if (n > cur) + do { emit_bmp(); } while (cur < n); - /** - * Unit (sub-bitmap) size. Must be a power of 2. + /* + * Emit partial unit. */ - static const size_t unit = sizeof bitmap * CHAR_BIT; + if (origin_begin & (unit - 1)) { + const uint64_t j = min(len, + (origin_begin & ~(unit - 1)) + unit - origin_begin); + for (i = origin_begin; i < origin_begin + j; i++) + bitmap |= 1ULL << (i & (unit - 1)); + if (j == len) + return; - void emit_bmp(bool omit_if_zero = false) { - if (!bitmap && omit_if_zero) - out_.write((const char*)&bitmap, sizeof bitmap); - bitmap = 0; - cur++; + emit_bmp(); + + len -= j; + origin_begin = i; } - }; + + /* + * Emit full units until end. + */ + n = (origin_begin + len) / unit; + while (cur < n) { + bitmap = ~0; + emit_bmp(); + len -= unit; + } + origin_begin = cur * unit; + + /* + * Emit final unit. + */ + for (i = origin_begin; i < origin_begin + len; i++) + bitmap |= 1ULL << (i & (unit - 1)); + } + + void binary_emitter::single_map(uint64_t origin_block, uint64_t, uint32_t) { + range_map(origin_block, 0, 0, 1); + } + + void binary_emitter::emit_bmp(bool omit_if_zero) { + if (!(!bitmap && omit_if_zero)) + out_.write((const char*)&bitmap, sizeof bitmap); + bitmap = 0; + cur++; + } } //---------------------------------------------------------------- extern "C" { emitter::ptr create_emitter(ostream &out) { - return emitter::ptr(new binary_emitter(out)); + return emitter::ptr(new tmakatos_emitter::binary_emitter(out)); } } diff --git a/contrib/tmakatos_emitter.h b/contrib/tmakatos_emitter.h new file mode 100644 index 0000000..51919b8 --- /dev/null +++ b/contrib/tmakatos_emitter.h @@ -0,0 +1,69 @@ +#ifndef _TMAKATOS_EMITTER_H_ +#define _TMAKATOS_EMITTER_H_ + +#include "thin-provisioning/emitter.h" +#include + +using namespace std; +using namespace thin_provisioning; + +namespace tmakatos_emitter { + class binary_emitter : public emitter { + public: + binary_emitter(ostream &out); + + void begin_superblock(string const &uuid, + uint64_t time, + uint64_t trans_id, + boost::optional flags, + boost::optional version, + uint32_t data_block_size, + uint64_t nr_data_blocks, + boost::optional metadata_snap); + + void end_superblock(); + + void begin_device(uint32_t dev_id, + uint64_t mapped_blocks, + uint64_t trans_id, + uint64_t creation_time, + uint64_t snap_time); + + void end_device(); + + void begin_named_mapping(string const &name); + + void end_named_mapping(); + + void identifier(string const &name); + + void range_map(uint64_t origin_begin, uint64_t, uint32_t, + uint64_t len); + + void single_map(uint64_t origin_block, uint64_t, uint32_t); + + private: + ostream &out_; + + /** + * The entire virtual block allocation bitmap is segmented into 64-bit + * sub-bitmaps (units). + */ + uint64_t bitmap; + + /* + * Pointer to the current sub-bitmap (unit) that has not yet been + * emitted. + */ + uint64_t cur; + + /** + * Unit (sub-bitmap) size. Must be a power of 2. + */ + static const size_t unit = sizeof bitmap * CHAR_BIT; + + void emit_bmp(bool omit_if_zero = false); + }; +} + +#endif /* _TMAKATOS_EMITTER_H_ */ From 431f2ab50aff9908892db127187f1dbd8fbdb46d Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Wed, 6 Apr 2016 14:15:07 +0300 Subject: [PATCH 090/118] allow tmakatos (a binary) emitter to be statically compiled It is trivial to implement static compilation of the rest of the 3rd party emitters. --- Makefile.in | 14 ++++++++++++- thin-provisioning/static_library_emitter.cc | 22 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 thin-provisioning/static_library_emitter.cc diff --git a/Makefile.in b/Makefile.in index 4373086..fed1756 100644 --- a/Makefile.in +++ b/Makefile.in @@ -92,7 +92,6 @@ SOURCE=\ thin-provisioning/restore_emitter.cc \ thin-provisioning/rmap_visitor.cc \ thin-provisioning/superblock.cc \ - thin-provisioning/shared_library_emitter.cc \ thin-provisioning/thin_check.cc \ thin-provisioning/thin_delta.cc \ thin-provisioning/thin_dump.cc \ @@ -119,10 +118,23 @@ ifeq ("@DEVTOOLS@", "yes") SOURCE+=$(DEVTOOLS_SOURCE) endif +ifeq ("@STATIC@", "yes") +SOURCE += thin-provisioning/static_library_emitter.cc +else +SOURCE += thin-provisioning/shared_library_emitter.cc +endif + CC:=@CC@ CXX:=@CXX@ STRIP:=@STRIP@ OBJECTS:=$(subst .cc,.o,$(SOURCE)) + +# FIXME OBJECTS += $(PLUGIN_LIBS) doesn't work, probably because it's empty at +# the time of use? +ifeq ("@STATIC@", "yes") +OBJECTS += contrib/*.a +endif + TOP_DIR:=@top_srcdir@ TOP_BUILDDIR:=@top_builddir@ CFLAGS+=-g -Wall -O3 -fPIC diff --git a/thin-provisioning/static_library_emitter.cc b/thin-provisioning/static_library_emitter.cc new file mode 100644 index 0000000..8caa507 --- /dev/null +++ b/thin-provisioning/static_library_emitter.cc @@ -0,0 +1,22 @@ +#include "thin-provisioning/shared_library_emitter.h" +#include +#include "contrib/tmakatos_emitter.h" + +using namespace std; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +emitter::ptr +thin_provisioning::create_custom_emitter(string const &shared_lib, ostream &out) +{ + if (shared_lib != "tmakatos_emitter.so") + throw runtime_error(shared_lib + ": no such emitter"); + + cout << "XXX creating tmakatos_emitter" << endl; + cout << flush; + + return emitter::ptr(new tmakatos_emitter::binary_emitter(out)); +} + +//---------------------------------------------------------------- From 5ebeb64cf89c26630e8de804fc49b4fd4bf00f4a Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Wed, 6 Apr 2016 15:15:58 +0300 Subject: [PATCH 091/118] don't avoid emitting part of the allocation bitmap when NOT told to do so The condition in the if statement is wrong, as what we're trying to achieve is to avoid emitting the particular piece of allocation bitmap if that piece is empty AND we specifically request to skip empty bitmaps. --- contrib/tmakatos_emitter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tmakatos_emitter.cc b/contrib/tmakatos_emitter.cc index 32ad397..c2b1118 100644 --- a/contrib/tmakatos_emitter.cc +++ b/contrib/tmakatos_emitter.cc @@ -111,7 +111,7 @@ namespace tmakatos_emitter { } void binary_emitter::emit_bmp(bool omit_if_zero) { - if (!(!bitmap && omit_if_zero)) + if (bitmap || !omit_if_zero) out_.write((const char*)&bitmap, sizeof bitmap); bitmap = 0; cur++; From c8fec7ec4055d0f4bcbca8e7d80c9ca419f3325c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 14 Apr 2016 08:54:32 +0100 Subject: [PATCH 092/118] [cache_writeback] Coded, needs testing --- Makefile.in | 3 + base/unique_handle.h | 67 ++++++++++++++ block-cache/copier.cc | 120 ++++++++++++++++++++++++- block-cache/copier.h | 64 ++++++++++++-- block-cache/io_engine.cc | 175 +++++++++++++++++++++++++++++++++++++ block-cache/io_engine.h | 82 +++++++++++++++++ block-cache/mem_pool.cc | 57 ++++++++++++ block-cache/mem_pool.h | 45 ++++++++++ caching/cache_writeback.cc | 19 +++- 9 files changed, 620 insertions(+), 12 deletions(-) create mode 100644 base/unique_handle.h create mode 100644 block-cache/io_engine.cc create mode 100644 block-cache/io_engine.h create mode 100644 block-cache/mem_pool.cc create mode 100644 block-cache/mem_pool.h diff --git a/Makefile.in b/Makefile.in index 87f2fd7..d2e1e97 100644 --- a/Makefile.in +++ b/Makefile.in @@ -36,6 +36,9 @@ SOURCE=\ base/rolling_hash.cc \ base/xml_utils.cc \ block-cache/block_cache.cc \ + block-cache/copier.cc \ + block-cache/io_engine.cc \ + block-cache/mem_pool.cc \ caching/cache_check.cc \ caching/cache_dump.cc \ caching/cache_metadata_size.cc \ diff --git a/base/unique_handle.h b/base/unique_handle.h new file mode 100644 index 0000000..9c44e1b --- /dev/null +++ b/base/unique_handle.h @@ -0,0 +1,67 @@ +#ifndef BASE_UNIQUE_HANDLE_H +#define BASE_UNIQUE_HANDLE_H + +#include +#include +#include + +//---------------------------------------------------------------- + +namespace base { + template + class unique_handle + { + public: + unique_handle(std::nullptr_t = nullptr) + : id_(TNul) { + } + + unique_handle(T x) + : id_(x) { + } + + explicit operator bool() const { + return id_ != TNul; + } + + operator T&() { + return id_; + } + + operator T() const { + return id_; + } + + T *operator&() { + return &id_; + } + + const T *operator&() const { + return &id_; + } + + friend bool operator == (unique_handle a, unique_handle b) { return a.id_ == b.id_; } + friend bool operator != (unique_handle a, unique_handle b) { return a.id_ != b.id_; } + friend bool operator == (unique_handle a, std::nullptr_t) { return a.id_ == TNul; } + friend bool operator != (unique_handle a, std::nullptr_t) { return a.id_ != TNul; } + friend bool operator == (std::nullptr_t, unique_handle b) { return TNul == b.id_; } + friend bool operator != (std::nullptr_t, unique_handle b) { return TNul != b.id_; } + + private: + T id_; + }; + + //-------------------------------- + + struct fd_deleter { + typedef unique_handle pointer; + void operator()(pointer p) { + ::close(p); + } + }; + typedef std::unique_ptr unique_fd; +} + +//---------------------------------------------------------------- + +#endif diff --git a/block-cache/copier.cc b/block-cache/copier.cc index 60ca93b..c130e14 100644 --- a/block-cache/copier.cc +++ b/block-cache/copier.cc @@ -1,7 +1,123 @@ #include "block-cache/copier.h" -//---------------------------------------------------------------- - +#include +using namespace bcache; +using namespace boost; +using namespace std; + +//---------------------------------------------------------------- + +copier::copier(string const &src, string const &dest, + sector_t block_size, size_t mem) + : pool_(block_size, mem), + block_size_(block_size), + nr_blocks_(mem / block_size), + engine_(nr_blocks_), + src_handle_(engine_.open_file(src, io_engine::READ_ONLY)), + dest_handle_(engine_.open_file(dest, io_engine::READ_WRITE)), + genkey_count_(0) +{ +} + +void +copier::issue(copy_op const &op) +{ + auto data = pool_.alloc(); + if (!data) { + wait_(); + data = pool_.alloc(); + + if (!data) + // Shouldn't get here + throw runtime_error("couldn't allocate buffer"); + } + + copy_job job(op, *data); + job.op.read_complete = job.op.write_complete = false; + unsigned key = genkey(); // used as context for the io_engine + + engine_.issue_io(src_handle_, + io_engine::READ, + to_sector(op.src_b), + to_sector(op.src_e), + *data, + key); + jobs_.insert(make_pair(key, job)); +} + +unsigned +copier::nr_pending() const +{ + return jobs_.size() + complete_.size(); +} + +boost::optional +copier::wait() +{ + while (complete_.empty() && !jobs_.empty()) + wait_(); + + if (complete_.empty()) + return optional(); + + else { + auto op = complete_.front(); + complete_.pop_front(); + return optional(op); + } +} + +void +copier::wait_() +{ + auto p = engine_.wait(); + auto it = jobs_.find(p.second); + if (it == jobs_.end()) + throw runtime_error("Internal error. Lost track of copy job."); + + copy_job j = it->second; + if (!p.first) { + // IO was unsuccessful + jobs_.erase(it); + complete(j); + return; + } + + // IO was successful + if (!j.op.read_complete) { + j.op.read_complete = true; + engine_.issue_io(dest_handle_, + io_engine::WRITE, + to_sector(j.op.dest_b), + to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)), + j.data, + it->first); + + } else { + j.op.write_complete = true; + jobs_.erase(it); + complete(j); + } +} + +void +copier::complete(copy_job const &j) +{ + pool_.free(j.data); + complete_.push_back(j.op); +} + +sector_t +copier::to_sector(block_address b) const +{ + return b * block_size_; +} + +unsigned +copier::genkey() +{ + return genkey_count_++; +} //---------------------------------------------------------------- diff --git a/block-cache/copier.h b/block-cache/copier.h index 41ed82e..28fdd8a 100644 --- a/block-cache/copier.h +++ b/block-cache/copier.h @@ -1,27 +1,75 @@ #ifndef BLOCK_CACHE_COPIER_H #define BLOCK_CACHE_COPIER_H -#include "block_cache.h" +#include "block-cache/io_engine.h" +#include "block-cache/mem_pool.h" #include +#include +#include //---------------------------------------------------------------- namespace bcache { + using block_address = uint64_t; + + struct copy_op { + copy_op() + : read_complete(false), + write_complete(false) { + } + + block_address src_b, src_e; + block_address dest_b; + + bool read_complete; + bool write_complete; + }; + + class copy_job { + public: + copy_job(copy_op const &op_, void *data_) + : op(op_), data(data_) { + } + + copy_op op; + void *data; + }; + class copier { public: - // block size in sectors copier(std::string const &src, std::string const &dest, - unsigned block_size); - ~copier(); + sector_t block_size, size_t mem); - // Returns the number of sectors copied - unsigned copy(block_address from, block_address to); + sector_t get_block_size() const { + return block_size_; + } - unsigned get_block_size() const; + // Blocks if out of memory. + void issue(copy_op const &op); + + unsigned nr_pending() const; + boost::optional wait(); private: - unsigned block_size_; + void wait_(); + void complete(copy_job const &j); + + sector_t to_sector(block_address b) const; + unsigned genkey(); + + mempool pool_; + sector_t block_size_; + unsigned nr_blocks_; + io_engine engine_; + io_engine::handle src_handle_; + io_engine::handle dest_handle_; + unsigned genkey_count_; + + using job_map = std::map; + using op_list = std::list; + job_map jobs_; + op_list complete_; }; } diff --git a/block-cache/io_engine.cc b/block-cache/io_engine.cc new file mode 100644 index 0000000..5580c8a --- /dev/null +++ b/block-cache/io_engine.cc @@ -0,0 +1,175 @@ +#include "base/container_of.h" +#include "block-cache/io_engine.h" + +#include +#include +#include +#include +#include +#include + +using namespace bcache; +using namespace boost; +using namespace std; + +#define SECTOR_SHIFT 9 + +//---------------------------------------------------------------- + +control_block_set::control_block_set(unsigned nr) + : cbs_(nr) +{ + for (auto i = 0u; i < nr; i++) + free_cbs_.insert(i); +} + +iocb * +control_block_set::alloc(unsigned context) +{ + if (free_cbs_.empty()) + return nullptr; + + auto it = free_cbs_.begin(); + + cblock &cb = cbs_[*it]; + cb.context = context; + free_cbs_.erase(it); + + return &cb.cb; +} + +void +control_block_set::free(iocb *cb) +{ + cblock *b = base::container_of(cb, &cblock::cb); + unsigned index = b - &cbs_[0]; + free_cbs_.insert(index); +} + +unsigned +control_block_set::context(iocb *cb) const +{ + cblock *b = base::container_of(cb, &cblock::cb); + return b->context; +} + +//---------------------------------------------------------------- + +io_engine::io_engine(unsigned max_io) + : aio_context_(0), + cbs_(max_io), + events_(max_io) +{ + int r = io_setup(max_io, &aio_context_); + if (r < 0) + throw runtime_error("io_setup failed"); +} + +io_engine::~io_engine() +{ + io_destroy(aio_context_); +} + +io_engine::handle +io_engine:: open_file(std::string const &path, mode m) +{ + int flags = (m == READ_ONLY) ? O_RDONLY : O_RDWR; + int fd = ::open(path.c_str(), O_DIRECT | flags); + if (fd < 0) { + ostringstream out; + out << "unable to open '" << path << "'"; + throw runtime_error(out.str()); + } + + descriptors_.push_back(base::unique_fd(fd)); + + return static_cast(fd); +} + +void +io_engine::close_file(handle h) +{ + for (auto it = descriptors_.begin(); it != descriptors_.end(); ++it) { + unsigned it_h = it->get(); + if (it_h == h) { + descriptors_.erase(it); + return; + } + } + + ostringstream out; + out << "unknown descriptor (" << h << ")"; + throw runtime_error(out.str()); +} + +bool +io_engine::issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context) +{ + auto cb = cbs_.alloc(context); + if (!cb) + return false; + + memset(cb, 0, sizeof(*cb)); + cb->aio_fildes = static_cast(h); + cb->u.c.buf = data; + cb->u.c.offset = b << SECTOR_SHIFT; + cb->u.c.nbytes = (e - b) << SECTOR_SHIFT; + + cb->aio_lio_opcode = (d == READ) ? IO_CMD_PREAD : IO_CMD_PWRITE; + + int r = io_submit(aio_context_, 1, &cb); + if (r != 1) { + std::ostringstream out; + out << "couldn't issue " + << ((d == READ) ? "READ" : "WRITE") + << " io: io_submit "; + if (r < 0) + out << "failed with " << r; + else + out << "succeeded, but queued no io"; + + throw std::runtime_error(out.str()); + } + + return true; +} + +std::pair +io_engine::wait() +{ + int r; + unsigned i; + + r = io_getevents(aio_context_, 1, events_.size(), &events_[0], NULL); + if (r < 0) { + std::ostringstream out; + out << "io_getevents failed: " << r; + throw std::runtime_error(out.str()); + } + + for (i = 0; i < static_cast(r); i++) { + io_event const &e = events_[i]; + iocb *cb = reinterpret_cast(e.obj); + unsigned context = cbs_.context(cb); + cbs_.free(cb); + + if (e.res == cb->u.c.nbytes) + return make_pair(true, context); + + else { + std::ostringstream out; + out << "io failed" + << ", e.res = " << e.res + << ", e.res2 = " << e.res2 + << ", offset = " << cb->u.c.offset + << ", nbytes = " << cb->u.c.nbytes; + return make_pair(false, context); + } + } + + + // shouldn't get here + return make_pair(false, 0); +} + +//---------------------------------------------------------------- diff --git a/block-cache/io_engine.h b/block-cache/io_engine.h new file mode 100644 index 0000000..07d97a8 --- /dev/null +++ b/block-cache/io_engine.h @@ -0,0 +1,82 @@ +#ifndef BLOCK_CACHE_IO_ENGINE_H +#define BLOCK_CACHE_IO_ENGINE_H + +#include "base/unique_handle.h" + +#include +#include +#include +#include +#include + +//---------------------------------------------------------------- + +namespace bcache { + using sector_t = uint64_t; + + //---------------- + + class control_block_set { + public: + control_block_set(unsigned nr); + + iocb *alloc(unsigned context); + void free(iocb *); + + unsigned context(iocb *) const; + + private: + struct cblock { + unsigned context; + struct iocb cb; + }; + + std::set free_cbs_; + std::vector cbs_; + }; + + //---------------- + + class io_engine { + public: + enum mode { + READ_ONLY, + READ_WRITE + }; + + enum dir { + READ, + WRITE + }; + + // max_io is the maximum nr of concurrent ios expected + io_engine(unsigned max_io); + ~io_engine(); + + using handle = unsigned; + + handle open_file(std::string const &path, mode m); + void close_file(handle h); + + // returns false if there are insufficient resources to + // queue the IO + bool issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context); + + // returns (success, context) + std::pair wait(); + + private: + std::list descriptors_; + + io_context_t aio_context_; + control_block_set cbs_; + std::vector events_; + + io_engine(io_engine const &) = delete; + io_engine &operator =(io_engine const &) = delete; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/block-cache/mem_pool.cc b/block-cache/mem_pool.cc new file mode 100644 index 0000000..f655fd0 --- /dev/null +++ b/block-cache/mem_pool.cc @@ -0,0 +1,57 @@ +#include "block-cache/mem_pool.h" + +#include + +using namespace bcache; +using namespace boost; +using namespace mempool_detail; + +#define PAGE_SIZE 4096 + +//---------------------------------------------------------------- + +mempool::mempool(size_t block_size, size_t total_mem) +{ + mem_ = alloc_aligned(total_mem, PAGE_SIZE); + + unsigned nr_blocks = total_mem / block_size; + for (auto i = 0u; i < nr_blocks; i++) + free(static_cast(mem_) + (block_size * i)); +} + +mempool::~mempool() +{ + ::free(mem_); +} + +boost::optional +mempool::alloc() +{ + if (free_.empty()) + return optional(); + + mempool_detail::alloc_block &b = free_.front(); + free_.pop_front(); + return optional(reinterpret_cast(&b)); +} + +void +mempool::free(void *data) +{ + mempool_detail::alloc_block *b = reinterpret_cast(data); + free_.push_front(*b); +} + +void * +mempool::alloc_aligned(size_t len, size_t alignment) +{ + void *result = NULL; + int r = posix_memalign(&result, alignment, len); + if (r) + return NULL; + + return result; +} + +//---------------------------------------------------------------- + diff --git a/block-cache/mem_pool.h b/block-cache/mem_pool.h new file mode 100644 index 0000000..61c1812 --- /dev/null +++ b/block-cache/mem_pool.h @@ -0,0 +1,45 @@ +#ifndef BLOCK_CACHE_MEM_POOL_H +#define BLOCK_CACHE_MEM_POOL_H + +#include +#include +#include + +namespace bi = boost::intrusive; + +//---------------------------------------------------------------- + +namespace bcache { + // FIXME: move to base? + + namespace mempool_detail { + struct alloc_block : public bi::list_base_hook<> { + }; + }; + + class mempool { + public: + mempool(size_t block_size, size_t total_mem); + ~mempool(); + + boost::optional alloc(); + void free(void *data); + + private: + static void *alloc_aligned(size_t len, size_t alignment); + + using block_list = bi::list; + + void *mem_; + block_list free_; + + //---------------- + + mempool(mempool const &) = delete; + mempool &operator =(mempool const &) = delete; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc index 042ae74..d81655a 100644 --- a/caching/cache_writeback.cc +++ b/caching/cache_writeback.cc @@ -19,8 +19,13 @@ using namespace std; namespace { struct flags { + flags() + : cache_size(1024 * 1024 * 128) { + } + using maybe_string = boost::optional; + size_t cache_size; maybe_string metadata_dev; maybe_string origin_dev; maybe_string fast_dev; @@ -41,13 +46,22 @@ namespace { if (only_dirty_ && !(m.flags_ & M_DIRTY)) return; - auto sectors_copied = copier_.copy(cblock, m.oblock_); + copy_op cop; + cop.src_b = cblock; + cop.src_e = cblock + 1ull; + cop.dest_b = m.oblock_; + + // blocks + copier_.issue(cop); stats_.blocks_issued++; + +#if 0 if (sectors_copied < block_size_) { stats_.blocks_failed++; stats_.sectors_failed += block_size_ - sectors_copied; } +#endif } struct copy_stats { @@ -107,7 +121,8 @@ namespace { block_manager<>::ptr bm = open_bm(*f.metadata_dev, block_manager<>::READ_ONLY); metadata md(bm, metadata::OPEN); - copier c(*f.fast_dev, *f.origin_dev, md.sb_.data_block_size); + copier c(*f.fast_dev, *f.origin_dev, + md.sb_.data_block_size, f.cache_size); copy_visitor cv(c, clean_shutdown(md)); ignore_damage_visitor dv; walk_mapping_array(*md.mappings_, cv, dv); From 4f542456001510051d973ba1830fef58f11e6945 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 21 Apr 2016 14:31:52 +0100 Subject: [PATCH 093/118] [block_t] add check_raw to mock --- unit-tests/block_t.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/unit-tests/block_t.cc b/unit-tests/block_t.cc index 1a61d2b..c1a3172 100644 --- a/unit-tests/block_t.cc +++ b/unit-tests/block_t.cc @@ -64,6 +64,7 @@ namespace { typedef boost::shared_ptr ptr; MOCK_CONST_METHOD2(check, void(void const *, block_address)); + MOCK_CONST_METHOD1(check_raw, bool(void const *data)); MOCK_CONST_METHOD2(prepare, void(void *, block_address)); }; From 73a69abfd2db7ed9ee08418af811b191930f65ea Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sat, 7 May 2016 11:47:40 +0100 Subject: [PATCH 094/118] [unit-tests] copier and mem_pool tests. More to come --- block-cache/copier.cc | 1 + block-cache/copier.h | 11 +++++ block-cache/mem_pool.cc | 17 ++++--- block-cache/mem_pool.h | 5 +- unit-tests/Makefile.in | 2 + unit-tests/copier_t.cc | 103 +++++++++++++++++++++++++++++++++++++++ unit-tests/mem_pool_t.cc | 97 ++++++++++++++++++++++++++++++++++++ 7 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 unit-tests/copier_t.cc create mode 100644 unit-tests/mem_pool_t.cc diff --git a/block-cache/copier.cc b/block-cache/copier.cc index c130e14..a1e9a5f 100644 --- a/block-cache/copier.cc +++ b/block-cache/copier.cc @@ -37,6 +37,7 @@ copier::issue(copy_op const &op) job.op.read_complete = job.op.write_complete = false; unsigned key = genkey(); // used as context for the io_engine + cerr << "data = " << &(*data) << "\n"; engine_.issue_io(src_handle_, io_engine::READ, to_sector(op.src_b), diff --git a/block-cache/copier.h b/block-cache/copier.h index 28fdd8a..24e1b38 100644 --- a/block-cache/copier.h +++ b/block-cache/copier.h @@ -19,6 +19,17 @@ namespace bcache { write_complete(false) { } + + copy_op(block_address src_b_, + block_address src_e_, + block_address dest_b_) + : src_b(src_b_), + src_e(src_e_), + dest_b(dest_b_), + read_complete(false), + write_complete(false) { + } + block_address src_b, src_e; block_address dest_b; diff --git a/block-cache/mem_pool.cc b/block-cache/mem_pool.cc index f655fd0..db5cc9e 100644 --- a/block-cache/mem_pool.cc +++ b/block-cache/mem_pool.cc @@ -1,18 +1,19 @@ #include "block-cache/mem_pool.h" +#include +#include #include using namespace bcache; using namespace boost; using namespace mempool_detail; - -#define PAGE_SIZE 4096 +using namespace std; //---------------------------------------------------------------- -mempool::mempool(size_t block_size, size_t total_mem) +mempool::mempool(size_t block_size, size_t total_mem, size_t alignment) { - mem_ = alloc_aligned(total_mem, PAGE_SIZE); + mem_ = alloc_aligned(total_mem, alignment); unsigned nr_blocks = total_mem / block_size; for (auto i = 0u; i < nr_blocks; i++) @@ -21,6 +22,7 @@ mempool::mempool(size_t block_size, size_t total_mem) mempool::~mempool() { + free_.clear(); ::free(mem_); } @@ -47,8 +49,11 @@ mempool::alloc_aligned(size_t len, size_t alignment) { void *result = NULL; int r = posix_memalign(&result, alignment, len); - if (r) - return NULL; + if (r) { + ostringstream out; + out << "posix_memalign failed: len = " << len << ", alignment = " << alignment << ", r = " << r << "\n"; + throw runtime_error(out.str()); + } return result; } diff --git a/block-cache/mem_pool.h b/block-cache/mem_pool.h index 61c1812..50328d3 100644 --- a/block-cache/mem_pool.h +++ b/block-cache/mem_pool.h @@ -13,13 +13,14 @@ namespace bcache { // FIXME: move to base? namespace mempool_detail { - struct alloc_block : public bi::list_base_hook<> { + struct alloc_block : public bi::list_base_hook> { }; }; class mempool { public: - mempool(size_t block_size, size_t total_mem); + // alignment must be a power of 2 + mempool(size_t block_size, size_t total_mem, size_t alignment = 8); ~mempool(); boost::optional alloc(); diff --git a/unit-tests/Makefile.in b/unit-tests/Makefile.in index 38f3d04..8746456 100644 --- a/unit-tests/Makefile.in +++ b/unit-tests/Makefile.in @@ -55,9 +55,11 @@ TEST_SOURCE=\ unit-tests/btree_counter_t.cc \ unit-tests/btree_damage_visitor_t.cc \ unit-tests/cache_superblock_t.cc \ + unit-tests/copier_t.cc \ unit-tests/damage_tracker_t.cc \ unit-tests/endian_t.cc \ unit-tests/error_state_t.cc \ + unit-tests/mem_pool_t.cc \ unit-tests/rmap_visitor_t.cc \ unit-tests/rolling_hash_t.cc \ unit-tests/run_set_t.cc \ diff --git a/unit-tests/copier_t.cc b/unit-tests/copier_t.cc new file mode 100644 index 0000000..b366ff8 --- /dev/null +++ b/unit-tests/copier_t.cc @@ -0,0 +1,103 @@ +// Copyright (C) 2016 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include "gmock/gmock.h" +#include "block-cache/copier.h" +#include "test_utils.h" + + +#include + +using namespace boost; +using namespace std; +using namespace test; +using namespace testing; + +//---------------------------------------------------------------- + +namespace { + class temp_file { + public: + temp_file(string const &name_base, unsigned meg_size) + : path_(gen_path(name_base)) { + + int fd = ::open(path_.c_str(), O_CREAT | O_RDWR, 0666); + if (fd < 0) + throw runtime_error("couldn't open file"); + + if (::fallocate(fd, 0, 0, 1024 * 1024 * meg_size)) + throw runtime_error("couldn't fallocate"); + + ::close(fd); + } + + ~temp_file() { + ::unlink(path_.c_str()); + } + + string const &get_path() const { + return path_; + } + + private: + static string gen_path(string const &base) { + return string("./") + base + string(".tmp"); + } + + string path_; + }; + + class CopierTests : public Test { + public: + CopierTests() + : src_file_("copy_src", 32), + dest_file_("copy_dest", 32), + copier_(src_file_.get_path(), + dest_file_.get_path(), + 64, 1 * 1024 * 1024) { + } + + copier &get_copier() { + return copier_; + } + + private: + temp_file src_file_; + temp_file dest_file_; + + copier copier_; + }; +} + +//---------------------------------------------------------------- + +TEST_F(CopierTests, empty_test) +{ + // Copy first block + copy_op op1(0, 1, 0); + get_copier().issue(op1); + auto mop = get_copier().wait(); + + if (mop) { + cerr << "op completed\n"; + } else { + cerr << "no op completed\n"; + } +} + +//---------------------------------------------------------------- diff --git a/unit-tests/mem_pool_t.cc b/unit-tests/mem_pool_t.cc new file mode 100644 index 0000000..b385f74 --- /dev/null +++ b/unit-tests/mem_pool_t.cc @@ -0,0 +1,97 @@ +// Copyright (C) 2016 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include "gmock/gmock.h" +#include "block-cache/mem_pool.h" +#include "test_utils.h" + +#include + +using namespace boost; +using namespace std; +using namespace test; +using namespace testing; + +//---------------------------------------------------------------- + +namespace { + class MempoolTests : public Test { + public: + bool aligned(void *data, size_t alignment) { + return (reinterpret_cast(data) % alignment) == 0; + } + + private: + + }; +} + +//---------------------------------------------------------------- + +TEST_F(MempoolTests, empty_test) +{ +} + +TEST_F(MempoolTests, create_destroy_cycle) +{ + for (size_t bs = 64; bs <= 512; bs *= 2) { + mempool mp(bs, 4 * 1024 * 1024, bs); + } +} + +TEST_F(MempoolTests, alignments_observed) +{ + for (size_t bs = 64; bs <= 512; bs *= 2) { + mempool mp(bs, 512 * 1024, bs); + + for (unsigned i = 0; i < 100; i++) { + auto md = mp.alloc(); + + if (!md) + throw runtime_error("couldn't alloc"); + + ASSERT_THAT(aligned(*md, bs), Eq(true)); + } + } +} + +TEST_F(MempoolTests, alloc_free_cycle) +{ + mempool mp(512, 512 * 1024, 512); + + for (unsigned i = 0; i < 10000; i++) { + auto md = mp.alloc(); + mp.free(*md); + } +} + +TEST_F(MempoolTests, exhaust_pool) +{ + mempool mp(512, 100 * 512, 512); + + for (unsigned i = 0; i < 100; i++) { + auto md = mp.alloc(); + + ASSERT_THAT(*md, NEq(0)); + } + + auto md = mp.alloc(); + ASSERT(*md, Eq(0)); +} + +//---------------------------------------------------------------- From 8147d798bea8718a26eca7305faefe686ed307d2 Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Mon, 9 May 2016 13:25:26 +0300 Subject: [PATCH 095/118] don't use implicit rule when statically compiling emitters --- Makefile.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile.in b/Makefile.in index fed1756..0b309c8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -129,10 +129,10 @@ CXX:=@CXX@ STRIP:=@STRIP@ OBJECTS:=$(subst .cc,.o,$(SOURCE)) -# FIXME OBJECTS += $(PLUGIN_LIBS) doesn't work, probably because it's empty at +# FIXME EMITTERS += $(PLUGIN_LIBS) doesn't work, probably because it's empty at # the time of use? ifeq ("@STATIC@", "yes") -OBJECTS += contrib/*.a +EMITTERS += contrib/*.a endif TOP_DIR:=@top_srcdir@ @@ -198,11 +198,11 @@ endif #---------------------------------------------------------------- -lib/libpdata.a: $(OBJECTS) +lib/libpdata.a: $(OBJECTS) $(EMITTERS) @echo " [AR] $<" - $(V)ar -rv $@ $(OBJECTS) > /dev/null 2>&1 + $(V)ar -rv $@ $(OBJECTS) $(EMITTERS) > /dev/null 2>&1 -bin/pdata_tools: $(OBJECTS) +bin/pdata_tools: $(OBJECTS) $(EMITTERS) @echo " [LD] $@" $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(CXXLIB) From 2203e2b513430f66debf617bdb8e97a768ac6db4 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 18 May 2016 01:21:37 +0800 Subject: [PATCH 096/118] [btree_node_checker] tidy up the code 1. Add const qualifiers 2. Add virtual destructor --- .../data-structures/btree_node_checker.cc | 22 ++++++++--------- .../data-structures/btree_node_checker.h | 24 ++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/persistent-data/data-structures/btree_node_checker.cc b/persistent-data/data-structures/btree_node_checker.cc index a6c77e0..0f50670 100644 --- a/persistent-data/data-structures/btree_node_checker.cc +++ b/persistent-data/data-structures/btree_node_checker.cc @@ -6,11 +6,11 @@ using persistent_data::btree_detail::btree_node_checker; //---------------------------------------------------------------- -btree_node_checker::error_type btree_node_checker::get_last_error() { +btree_node_checker::error_type btree_node_checker::get_last_error() const { return last_error_; } -std::string btree_node_checker::get_last_error_string() { +std::string btree_node_checker::get_last_error_string() const { switch (last_error_) { case BLOCK_NR_MISMATCH: return block_nr_mismatch_string(); @@ -39,7 +39,7 @@ void btree_node_checker::reset() { last_error_ = NO_ERROR; } -std::string btree_node_checker::block_nr_mismatch_string() { +std::string btree_node_checker::block_nr_mismatch_string() const { std::ostringstream out; out << "block number mismatch: actually " << error_location_ @@ -48,7 +48,7 @@ std::string btree_node_checker::block_nr_mismatch_string() { return out.str(); } -std::string btree_node_checker::value_sizes_mismatch_string() { +std::string btree_node_checker::value_sizes_mismatch_string() const { std::ostringstream out; out << "value size mismatch: expected " << error_value_sizes_[1] << ", but got " << error_value_sizes_[0] @@ -58,7 +58,7 @@ std::string btree_node_checker::value_sizes_mismatch_string() { return out.str(); } -std::string btree_node_checker::max_entries_too_large_string() { +std::string btree_node_checker::max_entries_too_large_string() const { std::ostringstream out; out << "max entries too large: " << error_max_entries_ << " (block " << error_location_ << ")"; @@ -66,7 +66,7 @@ std::string btree_node_checker::max_entries_too_large_string() { return out.str(); } -std::string btree_node_checker::max_entries_not_divisible_string() { +std::string btree_node_checker::max_entries_not_divisible_string() const { std::ostringstream out; out << "max entries is not divisible by 3: " << error_max_entries_ << " (block " << error_location_ << ")"; @@ -74,7 +74,7 @@ std::string btree_node_checker::max_entries_not_divisible_string() { return out.str(); } -std::string btree_node_checker::nr_entries_too_large_string() { +std::string btree_node_checker::nr_entries_too_large_string() const { std::ostringstream out; out << "bad nr_entries: " << error_nr_entries_ << " < " @@ -84,7 +84,7 @@ std::string btree_node_checker::nr_entries_too_large_string() { return out.str(); } -std::string btree_node_checker::nr_entries_too_small_string() { +std::string btree_node_checker::nr_entries_too_small_string() const { std::ostringstream out; out << "too few entries in btree_node: " << error_nr_entries_ @@ -96,7 +96,7 @@ std::string btree_node_checker::nr_entries_too_small_string() { return out.str(); } -std::string btree_node_checker::keys_out_of_order_string() { +std::string btree_node_checker::keys_out_of_order_string() const { std::ostringstream out; out << "keys are out of order, " << error_keys_[0] << " <= " << error_keys_[1] @@ -105,7 +105,7 @@ std::string btree_node_checker::keys_out_of_order_string() { return out.str(); } -std::string btree_node_checker::parent_key_mismatch_string() { +std::string btree_node_checker::parent_key_mismatch_string() const { std::ostringstream out; out << "parent key mismatch: parent was " << error_keys_[1] << ", but lowest in node was " << error_keys_[0] @@ -114,7 +114,7 @@ std::string btree_node_checker::parent_key_mismatch_string() { return out.str(); } -std::string btree_node_checker::leaf_key_overlapped_string() { +std::string btree_node_checker::leaf_key_overlapped_string() const { std::ostringstream out; out << "the last key of the previous leaf was " << error_keys_[1] << " and the first key of this leaf is " << error_keys_[0] diff --git a/persistent-data/data-structures/btree_node_checker.h b/persistent-data/data-structures/btree_node_checker.h index 926f613..be50007 100644 --- a/persistent-data/data-structures/btree_node_checker.h +++ b/persistent-data/data-structures/btree_node_checker.h @@ -41,6 +41,8 @@ namespace persistent_data { error_keys_{0, 0} { } + virtual ~btree_node_checker() {} + template bool check_block_nr(btree_detail::node_ref const &n) { if (n.get_location() != n.get_block_nr()) { @@ -175,20 +177,20 @@ namespace persistent_data { return true; } - error_type get_last_error(); - std::string get_last_error_string(); + error_type get_last_error() const; + std::string get_last_error_string() const; void reset(); private: - std::string block_nr_mismatch_string(); - std::string value_sizes_mismatch_string(); - std::string max_entries_too_large_string(); - std::string max_entries_not_divisible_string(); - std::string nr_entries_too_large_string(); - std::string nr_entries_too_small_string(); - std::string keys_out_of_order_string(); - std::string parent_key_mismatch_string(); - std::string leaf_key_overlapped_string(); + std::string block_nr_mismatch_string() const; + std::string value_sizes_mismatch_string() const; + std::string max_entries_too_large_string() const; + std::string max_entries_not_divisible_string() const; + std::string nr_entries_too_large_string() const; + std::string nr_entries_too_small_string() const; + std::string keys_out_of_order_string() const; + std::string parent_key_mismatch_string() const; + std::string leaf_key_overlapped_string() const; error_type last_error_; block_address error_location_; From 55ecf87439ef290931d0db87dfc70ce01e37d88e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 18 May 2016 01:23:26 +0800 Subject: [PATCH 097/118] [thin_ll_restore] fix the data type of blocknr to uint64_t --- thin-provisioning/thin_ll_restore.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/thin_ll_restore.cc b/thin-provisioning/thin_ll_restore.cc index 27ab184..ee1d88f 100644 --- a/thin-provisioning/thin_ll_restore.cc +++ b/thin-provisioning/thin_ll_restore.cc @@ -87,7 +87,7 @@ namespace { device_tree_detail::device_details details; device_tree::ptr details_tree; - boost::optional details_root = get_opt_attr(attr, "blocknr"); + boost::optional details_root = get_opt_attr(attr, "blocknr"); if (details_root) details_tree = device_tree::ptr(new device_tree(*md->tm_, *details_root, device_tree_detail::device_details_traits::ref_counter())); @@ -112,7 +112,7 @@ namespace { } void parse_node(metadata::ptr md, emitter::ptr e, attributes const &attr) { - metadata_dump_subtree(md, e, true, get_attr(attr, "blocknr")); + metadata_dump_subtree(md, e, true, get_attr(attr, "blocknr")); } void start_tag(void *data, char const *el, char const **attr) { From e985b8b3be62e4cfda7f29c611c36a08a938d0e1 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 19 May 2016 00:47:54 +0800 Subject: [PATCH 098/118] [thin_scan][thin_ll_*] fix the data type for getopt_long return value to int For toolchains represent char as unsigned type --- thin-provisioning/thin_ll_dump.cc | 2 +- thin-provisioning/thin_ll_restore.cc | 2 +- thin-provisioning/thin_scan.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/thin-provisioning/thin_ll_dump.cc b/thin-provisioning/thin_ll_dump.cc index f16a745..94f6a08 100644 --- a/thin-provisioning/thin_ll_dump.cc +++ b/thin-provisioning/thin_ll_dump.cc @@ -330,7 +330,7 @@ thin_ll_dump_cmd::run(int argc, char **argv) boost::optional output; flags f; - char c; + int c; while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { switch(c) { case 'h': diff --git a/thin-provisioning/thin_ll_restore.cc b/thin-provisioning/thin_ll_restore.cc index ee1d88f..4082e0b 100644 --- a/thin-provisioning/thin_ll_restore.cc +++ b/thin-provisioning/thin_ll_restore.cc @@ -216,7 +216,7 @@ thin_ll_restore_cmd::run(int argc, char **argv) { string output; string input_metadata; flags f; - char c; + int c; const char shortopts[] = "hi:o:E:V"; const struct option longopts[] = { diff --git a/thin-provisioning/thin_scan.cc b/thin-provisioning/thin_scan.cc index 1ed7a6b..6f63df0 100644 --- a/thin-provisioning/thin_scan.cc +++ b/thin-provisioning/thin_scan.cc @@ -383,7 +383,7 @@ thin_scan_cmd::run(int argc, char **argv) boost::optional output; flags f; - char c; + int c; while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { switch(c) { case 'h': From 34c039d7dc003c3fdb71285785382d0f679f4648 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 1 Jun 2016 14:46:27 +0100 Subject: [PATCH 099/118] [mempool] more tests --- block-cache/copier.cc | 6 +++--- block-cache/mem_pool.cc | 6 +++--- block-cache/mem_pool.h | 2 +- unit-tests/mem_pool_t.cc | 9 ++++----- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/block-cache/copier.cc b/block-cache/copier.cc index a1e9a5f..192c0a1 100644 --- a/block-cache/copier.cc +++ b/block-cache/copier.cc @@ -33,16 +33,16 @@ copier::issue(copy_op const &op) throw runtime_error("couldn't allocate buffer"); } - copy_job job(op, *data); + copy_job job(op, data); job.op.read_complete = job.op.write_complete = false; unsigned key = genkey(); // used as context for the io_engine - cerr << "data = " << &(*data) << "\n"; + cerr << "data = " << data << "\n"; engine_.issue_io(src_handle_, io_engine::READ, to_sector(op.src_b), to_sector(op.src_e), - *data, + data, key); jobs_.insert(make_pair(key, job)); } diff --git a/block-cache/mem_pool.cc b/block-cache/mem_pool.cc index db5cc9e..03f9455 100644 --- a/block-cache/mem_pool.cc +++ b/block-cache/mem_pool.cc @@ -26,15 +26,15 @@ mempool::~mempool() ::free(mem_); } -boost::optional +void * mempool::alloc() { if (free_.empty()) - return optional(); + return nullptr; mempool_detail::alloc_block &b = free_.front(); free_.pop_front(); - return optional(reinterpret_cast(&b)); + return reinterpret_cast(&b); } void diff --git a/block-cache/mem_pool.h b/block-cache/mem_pool.h index 50328d3..276a314 100644 --- a/block-cache/mem_pool.h +++ b/block-cache/mem_pool.h @@ -23,7 +23,7 @@ namespace bcache { mempool(size_t block_size, size_t total_mem, size_t alignment = 8); ~mempool(); - boost::optional alloc(); + void *alloc(); void free(void *data); private: diff --git a/unit-tests/mem_pool_t.cc b/unit-tests/mem_pool_t.cc index b385f74..24fb34c 100644 --- a/unit-tests/mem_pool_t.cc +++ b/unit-tests/mem_pool_t.cc @@ -65,7 +65,7 @@ TEST_F(MempoolTests, alignments_observed) if (!md) throw runtime_error("couldn't alloc"); - ASSERT_THAT(aligned(*md, bs), Eq(true)); + ASSERT_TRUE(aligned(md, bs)); } } } @@ -76,7 +76,7 @@ TEST_F(MempoolTests, alloc_free_cycle) for (unsigned i = 0; i < 10000; i++) { auto md = mp.alloc(); - mp.free(*md); + mp.free(md); } } @@ -86,12 +86,11 @@ TEST_F(MempoolTests, exhaust_pool) for (unsigned i = 0; i < 100; i++) { auto md = mp.alloc(); - - ASSERT_THAT(*md, NEq(0)); + ASSERT_NE(md, nullptr); } auto md = mp.alloc(); - ASSERT(*md, Eq(0)); + ASSERT_EQ(md, nullptr); } //---------------------------------------------------------------- From a94bfea79853adf177584003f80704e923452b73 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 7 Jun 2016 11:12:27 +0100 Subject: [PATCH 100/118] [block-cache] unit tests + debug io_engine and copier --- Makefile.in | 4 +- block-cache/copier.cc | 55 ++++++---- block-cache/copier.h | 16 ++- block-cache/io_engine.cc | 81 +++++++------- block-cache/io_engine.h | 71 ++++++++----- caching/cache_writeback.cc | 4 +- unit-tests/Makefile.in | 19 ++-- unit-tests/copier_t.cc | 213 +++++++++++++++++++++++++++++-------- unit-tests/io_engine_t.cc | 185 ++++++++++++++++++++++++++++++++ unit-tests/mem_pool_t.cc | 16 +++ unit-tests/test_utils.cc | 34 ++++++ unit-tests/test_utils.h | 12 +++ 12 files changed, 560 insertions(+), 150 deletions(-) create mode 100644 unit-tests/io_engine_t.cc diff --git a/Makefile.in b/Makefile.in index d2e1e97..1fa391b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -166,8 +166,8 @@ INSTALL_DATA = $(INSTALL) -p -m 644 ifeq ("@TESTING@", "yes") TEST_INCLUDES=\ - -Igmock-1.6.0/include \ - -Igmock-1.6.0/gtest/include + -Igoogletest/googlemock/include \ + -Igoogletest/googletest/include else TEST_INCLUDES= endif diff --git a/block-cache/copier.cc b/block-cache/copier.cc index 192c0a1..9910b58 100644 --- a/block-cache/copier.cc +++ b/block-cache/copier.cc @@ -8,18 +8,25 @@ using namespace std; //---------------------------------------------------------------- -copier::copier(string const &src, string const &dest, +copier::copier(io_engine &engine, + string const &src, string const &dest, sector_t block_size, size_t mem) - : pool_(block_size, mem), + : pool_(block_size * 512, mem), block_size_(block_size), nr_blocks_(mem / block_size), - engine_(nr_blocks_), + engine_(engine), src_handle_(engine_.open_file(src, io_engine::READ_ONLY)), dest_handle_(engine_.open_file(dest, io_engine::READ_WRITE)), genkey_count_(0) { } +copier::~copier() +{ + engine_.close_file(src_handle_); + engine_.close_file(dest_handle_); +} + void copier::issue(copy_op const &op) { @@ -37,14 +44,17 @@ copier::issue(copy_op const &op) job.op.read_complete = job.op.write_complete = false; unsigned key = genkey(); // used as context for the io_engine - cerr << "data = " << data << "\n"; - engine_.issue_io(src_handle_, - io_engine::READ, - to_sector(op.src_b), - to_sector(op.src_e), - data, - key); - jobs_.insert(make_pair(key, job)); + auto r = engine_.issue_io(src_handle_, + io_engine::READ, + to_sector(op.src_b), + to_sector(op.src_e), + data, + key); + + if (r) + jobs_.insert(make_pair(key, job)); + else + complete(job); } unsigned @@ -56,7 +66,7 @@ copier::nr_pending() const boost::optional copier::wait() { - while (complete_.empty() && !jobs_.empty()) + while (!jobs_.empty() && complete_.empty()) wait_(); if (complete_.empty()) @@ -77,28 +87,31 @@ copier::wait_() if (it == jobs_.end()) throw runtime_error("Internal error. Lost track of copy job."); - copy_job j = it->second; + copy_job &j = it->second; if (!p.first) { // IO was unsuccessful - jobs_.erase(it); complete(j); + jobs_.erase(it); return; } // IO was successful if (!j.op.read_complete) { j.op.read_complete = true; - engine_.issue_io(dest_handle_, - io_engine::WRITE, - to_sector(j.op.dest_b), - to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)), - j.data, - it->first); + if (!engine_.issue_io(dest_handle_, + io_engine::WRITE, + to_sector(j.op.dest_b), + to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)), + j.data, + it->first)) { + complete(j); + jobs_.erase(it); + } } else { j.op.write_complete = true; - jobs_.erase(it); complete(j); + jobs_.erase(it); } } diff --git a/block-cache/copier.h b/block-cache/copier.h index 24e1b38..5eb2a51 100644 --- a/block-cache/copier.h +++ b/block-cache/copier.h @@ -15,11 +15,13 @@ namespace bcache { struct copy_op { copy_op() - : read_complete(false), + : src_b(0), + src_e(0), + dest_b(0), + read_complete(false), write_complete(false) { } - copy_op(block_address src_b_, block_address src_e_, block_address dest_b_) @@ -30,6 +32,10 @@ namespace bcache { write_complete(false) { } + bool success() const { + return read_complete && write_complete; + } + block_address src_b, src_e; block_address dest_b; @@ -49,8 +55,10 @@ namespace bcache { class copier { public: - copier(std::string const &src, std::string const &dest, + copier(io_engine &engine, + std::string const &src, std::string const &dest, sector_t block_size, size_t mem); + ~copier(); sector_t get_block_size() const { return block_size_; @@ -72,7 +80,7 @@ namespace bcache { mempool pool_; sector_t block_size_; unsigned nr_blocks_; - io_engine engine_; + io_engine &engine_; io_engine::handle src_handle_; io_engine::handle dest_handle_; unsigned genkey_count_; diff --git a/block-cache/io_engine.cc b/block-cache/io_engine.cc index 5580c8a..7837c33 100644 --- a/block-cache/io_engine.cc +++ b/block-cache/io_engine.cc @@ -12,7 +12,12 @@ using namespace bcache; using namespace boost; using namespace std; -#define SECTOR_SHIFT 9 +//---------------------------------------------------------------- + +namespace { + unsigned const SECTOR_SHIFT = 9; + unsigned const PAGE_SIZE = 4096; +} //---------------------------------------------------------------- @@ -55,23 +60,22 @@ control_block_set::context(iocb *cb) const //---------------------------------------------------------------- -io_engine::io_engine(unsigned max_io) +aio_engine::aio_engine(unsigned max_io) : aio_context_(0), - cbs_(max_io), - events_(max_io) + cbs_(max_io) { int r = io_setup(max_io, &aio_context_); if (r < 0) throw runtime_error("io_setup failed"); } -io_engine::~io_engine() +aio_engine::~aio_engine() { io_destroy(aio_context_); } -io_engine::handle -io_engine:: open_file(std::string const &path, mode m) +aio_engine::handle +aio_engine::open_file(std::string const &path, mode m) { int flags = (m == READ_ONLY) ? O_RDONLY : O_RDWR; int fd = ::open(path.c_str(), O_DIRECT | flags); @@ -87,7 +91,7 @@ io_engine:: open_file(std::string const &path, mode m) } void -io_engine::close_file(handle h) +aio_engine::close_file(handle h) { for (auto it = descriptors_.begin(); it != descriptors_.end(); ++it) { unsigned it_h = it->get(); @@ -103,71 +107,60 @@ io_engine::close_file(handle h) } bool -io_engine::issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context) +aio_engine::issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context) { - auto cb = cbs_.alloc(context); + if (reinterpret_cast(data) & (PAGE_SIZE - 1)) + throw runtime_error("Data passed to issue_io must be page aligned\n"); + + iocb *cb; + + cb = cbs_.alloc(context); if (!cb) return false; memset(cb, 0, sizeof(*cb)); + cb->aio_fildes = static_cast(h); cb->u.c.buf = data; cb->u.c.offset = b << SECTOR_SHIFT; cb->u.c.nbytes = (e - b) << SECTOR_SHIFT; - cb->aio_lio_opcode = (d == READ) ? IO_CMD_PREAD : IO_CMD_PWRITE; int r = io_submit(aio_context_, 1, &cb); - if (r != 1) { - std::ostringstream out; - out << "couldn't issue " - << ((d == READ) ? "READ" : "WRITE") - << " io: io_submit "; - if (r < 0) - out << "failed with " << r; - else - out << "succeeded, but queued no io"; - - throw std::runtime_error(out.str()); - } - - return true; + return r == 1; } std::pair -io_engine::wait() +aio_engine::wait() { int r; - unsigned i; + struct io_event event; - r = io_getevents(aio_context_, 1, events_.size(), &events_[0], NULL); + memset(&event, 0, sizeof(event)); + + r = io_getevents(aio_context_, 1, 1, &event, NULL); if (r < 0) { std::ostringstream out; out << "io_getevents failed: " << r; throw std::runtime_error(out.str()); } - for (i = 0; i < static_cast(r); i++) { - io_event const &e = events_[i]; - iocb *cb = reinterpret_cast(e.obj); - unsigned context = cbs_.context(cb); + iocb *cb = reinterpret_cast(event.obj); + unsigned context = cbs_.context(cb); + + if (event.res == cb->u.c.nbytes) { cbs_.free(cb); + return make_pair(true, context); - if (e.res == cb->u.c.nbytes) - return make_pair(true, context); + } else if (static_cast(event.res) < 0) { + cbs_.free(cb); + return make_pair(false, context); - else { - std::ostringstream out; - out << "io failed" - << ", e.res = " << e.res - << ", e.res2 = " << e.res2 - << ", offset = " << cb->u.c.offset - << ", nbytes = " << cb->u.c.nbytes; - return make_pair(false, context); - } + } else { + cbs_.free(cb); + return make_pair(false, context); } - // shouldn't get here return make_pair(false, 0); } diff --git a/block-cache/io_engine.h b/block-cache/io_engine.h index 07d97a8..55b556d 100644 --- a/block-cache/io_engine.h +++ b/block-cache/io_engine.h @@ -5,16 +5,50 @@ #include #include -#include #include #include +#include //---------------------------------------------------------------- namespace bcache { using sector_t = uint64_t; - //---------------- + // Virtual base class to aid unit testing + class io_engine { + public: + enum mode { + READ_ONLY, + READ_WRITE + }; + + enum dir { + READ, + WRITE + }; + + io_engine() {} + virtual ~io_engine() {} + + using handle = unsigned; + + virtual handle open_file(std::string const &path, mode m) = 0; + virtual void close_file(handle h) = 0; + + // returns false if there are insufficient resources to + // queue the IO + virtual bool issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context) = 0; + + // returns (success, context) + using wait_result = std::pair; + virtual wait_result wait() = 0; + + private: + io_engine(io_engine const &) = delete; + io_engine &operator =(io_engine const &) = delete; + }; + + //-------------------------------- class control_block_set { public: @@ -37,43 +71,32 @@ namespace bcache { //---------------- - class io_engine { + class aio_engine : public io_engine { public: - enum mode { - READ_ONLY, - READ_WRITE - }; - - enum dir { - READ, - WRITE - }; - // max_io is the maximum nr of concurrent ios expected - io_engine(unsigned max_io); - ~io_engine(); + aio_engine(unsigned max_io); + ~aio_engine(); using handle = unsigned; - handle open_file(std::string const &path, mode m); - void close_file(handle h); + // FIXME: open exclusive? + virtual handle open_file(std::string const &path, mode m); + virtual void close_file(handle h); - // returns false if there are insufficient resources to - // queue the IO - bool issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context); + // Returns false if queueing the io failed + virtual bool issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context); // returns (success, context) - std::pair wait(); + virtual wait_result wait(); private: std::list descriptors_; io_context_t aio_context_; control_block_set cbs_; - std::vector events_; - io_engine(io_engine const &) = delete; - io_engine &operator =(io_engine const &) = delete; + aio_engine(io_engine const &) = delete; + aio_engine &operator =(io_engine const &) = delete; }; } diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc index d81655a..f7b9a28 100644 --- a/caching/cache_writeback.cc +++ b/caching/cache_writeback.cc @@ -120,8 +120,8 @@ namespace { int writeback_(flags const &f) { block_manager<>::ptr bm = open_bm(*f.metadata_dev, block_manager<>::READ_ONLY); metadata md(bm, metadata::OPEN); - - copier c(*f.fast_dev, *f.origin_dev, + aio_engine engine(f.cache_size / md.sb_.data_block_size); + copier c(engine, *f.fast_dev, *f.origin_dev, md.sb_.data_block_size, f.cache_size); copy_visitor cv(c, clean_shutdown(md)); ignore_damage_visitor dv; diff --git a/unit-tests/Makefile.in b/unit-tests/Makefile.in index 8746456..cee3a70 100644 --- a/unit-tests/Makefile.in +++ b/unit-tests/Makefile.in @@ -16,10 +16,10 @@ # with thin-provisioning-tools. If not, see # . -GMOCK_DIR=gmock-1.6.0/ +GMOCK_DIR=googletest GMOCK_INCLUDES=\ - -Igmock-1.6.0/include \ - -Igmock-1.6.0/gtest/include + -I$(GMOCK_DIR)/googlemock/include \ + -I$(GMOCK_DIR)/googletest/include GMOCK_FLAGS=\ -Wno-unused-local-typedefs @@ -28,16 +28,16 @@ GMOCK_LIBS=\ -Llib -lpdata -lgmock -lpthread -laio GMOCK_DEPS=\ - $(wildcard $(GMOCK_DIR)/include/*.h) \ - $(wildcard $(GMOCK_DIR)/src/*.cc) \ - $(wildcard $(GMOCK_DIR)/gtest/include/*.h) \ - $(wildcard $(GMOCK_DIR)/gtest/src/*.cc) + $(wildcard $(GMOCK_DIR)/googlemock/include/*.h) \ + $(wildcard $(GMOCK_DIR)/googlemock/src/*.cc) \ + $(wildcard $(GMOCK_DIR)/googletest/include/*.h) \ + $(wildcard $(GMOCK_DIR)/googletest/src/*.cc) lib/libgmock.a: $(GMOCK_DEPS) @echo " [CXX] gtest" - $(V)g++ $(GMOCK_INCLUDES) -I$(GMOCK_DIR)/gtest -c $(GMOCK_DIR)/gtest/src/gtest-all.cc + $(V)g++ $(GMOCK_INCLUDES) -I$(GMOCK_DIR)/googletest -c $(GMOCK_DIR)/googletest/src/gtest-all.cc @echo " [CXX] gmock" - $(V)g++ $(GMOCK_INCLUDES) -I$(GMOCK_DIR) -c $(GMOCK_DIR)/src/gmock-all.cc + $(V)g++ $(GMOCK_INCLUDES) -I$(GMOCK_DIR)/googlemock -c $(GMOCK_DIR)/googlemock/src/gmock-all.cc @echo " [AR] $<" $(V)ar -rv lib/libgmock.a gtest-all.o gmock-all.o > /dev/null 2>&1 @@ -59,6 +59,7 @@ TEST_SOURCE=\ unit-tests/damage_tracker_t.cc \ unit-tests/endian_t.cc \ unit-tests/error_state_t.cc \ + unit-tests/io_engine_t.cc \ unit-tests/mem_pool_t.cc \ unit-tests/rmap_visitor_t.cc \ unit-tests/rolling_hash_t.cc \ diff --git a/unit-tests/copier_t.cc b/unit-tests/copier_t.cc index b366ff8..e95819f 100644 --- a/unit-tests/copier_t.cc +++ b/unit-tests/copier_t.cc @@ -31,72 +31,197 @@ using namespace testing; //---------------------------------------------------------------- namespace { - class temp_file { + class io_engine_mock : public io_engine { public: - temp_file(string const &name_base, unsigned meg_size) - : path_(gen_path(name_base)) { + MOCK_METHOD2(open_file, handle(string const &, mode)); + MOCK_METHOD1(close_file, void(handle)); + MOCK_METHOD6(issue_io, bool(handle, dir, sector_t, sector_t, void *, unsigned)); - int fd = ::open(path_.c_str(), O_CREAT | O_RDWR, 0666); - if (fd < 0) - throw runtime_error("couldn't open file"); - - if (::fallocate(fd, 0, 0, 1024 * 1024 * meg_size)) - throw runtime_error("couldn't fallocate"); - - ::close(fd); - } - - ~temp_file() { - ::unlink(path_.c_str()); - } - - string const &get_path() const { - return path_; - } - - private: - static string gen_path(string const &base) { - return string("./") + base + string(".tmp"); - } - - string path_; + MOCK_METHOD0(wait, wait_result()); }; + unsigned const BLOCK_SIZE = 64u; + using wait_result = io_engine::wait_result; + class CopierTests : public Test { public: CopierTests() - : src_file_("copy_src", 32), - dest_file_("copy_dest", 32), - copier_(src_file_.get_path(), - dest_file_.get_path(), - 64, 1 * 1024 * 1024) { + : src_file_("copy_src"), + dest_file_("copy_dest") { } - copier &get_copier() { - return copier_; + unique_ptr make_copier() { + EXPECT_CALL(engine_, open_file(src_file_, io_engine::READ_ONLY)). + WillOnce(Return(SRC_HANDLE)); + EXPECT_CALL(engine_, open_file(dest_file_, io_engine::READ_WRITE)). + WillOnce(Return(DEST_HANDLE)); + + EXPECT_CALL(engine_, close_file(SRC_HANDLE)).Times(1); + EXPECT_CALL(engine_, close_file(DEST_HANDLE)).Times(1); + + return unique_ptr(new copier(engine_, src_file_, + dest_file_, + BLOCK_SIZE, 1 * 1024 * 1024)); } - private: - temp_file src_file_; - temp_file dest_file_; + void issue_successful_op(copier &c, copy_op &op, unsigned context) { + InSequence dummy; - copier copier_; + unsigned nr_pending = c.nr_pending(); + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, + op.src_b * BLOCK_SIZE, + op.src_e * BLOCK_SIZE, _, context)). + WillOnce(Return(true)); + c.issue(op); + + ASSERT_TRUE(c.nr_pending() == nr_pending + 1); + + EXPECT_CALL(engine_, wait()). + WillOnce(Return(wait_result(true, context))); + + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::WRITE, + op.dest_b * BLOCK_SIZE, + (op.dest_b + (op.src_e - op.src_b)) * BLOCK_SIZE, _, context)). + WillOnce(Return(true)); + + EXPECT_CALL(engine_, wait()). + WillOnce(Return(wait_result(true, context))); + + auto mop = c.wait(); + ASSERT_EQ(c.nr_pending(), nr_pending); + + ASSERT_TRUE(mop->success()); + } + + unsigned const SRC_HANDLE = 10; + unsigned const DEST_HANDLE = 11; + + string src_file_; + string dest_file_; + StrictMock engine_; }; } //---------------------------------------------------------------- TEST_F(CopierTests, empty_test) +{ + auto c = make_copier(); +} + +TEST_F(CopierTests, successful_copy) +{ + // Copy first block + copy_op op1(0, 1, 0); + + auto c = make_copier(); + issue_successful_op(*c, op1, 0); +} + +TEST_F(CopierTests, unsuccessful_issue_read) +{ + copy_op op1(0, 1, 0); + auto c = make_copier(); + + InSequence dummy; + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(false)); + c->issue(op1); + + ASSERT_EQ(c->nr_pending(), 1u); + + auto mop = c->wait(); + ASSERT_EQ(c->nr_pending(), 0u); + ASSERT_FALSE(mop->success()); +} + +TEST_F(CopierTests, unsuccessful_read) +{ + copy_op op1(0, 1, 0); + auto c = make_copier(); + + InSequence dummy; + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(true)); + c->issue(op1); + + ASSERT_EQ(c->nr_pending(), 1u); + EXPECT_CALL(engine_, wait()). + WillOnce(Return(wait_result(false, 0u))); + ASSERT_EQ(c->nr_pending(), 1u); + + auto mop = c->wait(); + ASSERT_EQ(c->nr_pending(), 0u); + ASSERT_FALSE(mop->success()); +} + +TEST_F(CopierTests, unsuccessful_issue_write) +{ + copy_op op1(0, 1, 0); + auto c = make_copier(); + + InSequence dummy; + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(true)); + c->issue(op1); + + ASSERT_EQ(c->nr_pending(), 1u); + + EXPECT_CALL(engine_, wait()). + WillOnce(Return(wait_result(true, 0u))); + ASSERT_EQ(c->nr_pending(), 1u); + + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::WRITE, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(false)); + + auto mop = c->wait(); + ASSERT_EQ(c->nr_pending(), 0u); + ASSERT_FALSE(mop->success()); +} + +TEST_F(CopierTests, unsuccessful_write) { // Copy first block copy_op op1(0, 1, 0); - get_copier().issue(op1); - auto mop = get_copier().wait(); - if (mop) { - cerr << "op completed\n"; - } else { - cerr << "no op completed\n"; + auto c = make_copier(); + + InSequence dummy; + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(true)); + c->issue(op1); + ASSERT_EQ(c->nr_pending(), 1u); + + EXPECT_CALL(engine_, wait()). + WillOnce(Return(wait_result(true, 0u))); + + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::WRITE, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(true)); + + EXPECT_CALL(engine_, wait()). + WillOnce(Return(wait_result(false, 0u))); + + auto mop = c->wait(); + ASSERT_EQ(c->nr_pending(), 0u); + + ASSERT_FALSE(mop->success()); +} + +TEST_F(CopierTests, copy_the_same_block_many_times) +{ + auto c = make_copier(); + copy_op op1(0, 1, 0); + + for (unsigned i = 0; i < 50000; i++) + issue_successful_op(*c, op1, i); +} + +TEST_F(CopierTests, copy_different_blocks) +{ + auto c = make_copier(); + for (unsigned i = 0; i < 5000; i++) { + copy_op op(i, i + 1, i); + issue_successful_op(*c, op, i); } } diff --git a/unit-tests/io_engine_t.cc b/unit-tests/io_engine_t.cc new file mode 100644 index 0000000..f1b3a9e --- /dev/null +++ b/unit-tests/io_engine_t.cc @@ -0,0 +1,185 @@ +// Copyright (C) 2016 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include "gmock/gmock.h" +#include "block-cache/mem_pool.h" +#include "block-cache/io_engine.h" +#include "test_utils.h" + + +#include + +using namespace boost; +using namespace std; +using namespace test; +using namespace testing; + +//---------------------------------------------------------------- + +namespace { + unsigned const MAX_IO = 64; + unsigned const PAGE_SIZE = 4096; + + class IOEngineTests : public Test { + public: + IOEngineTests() + : pool_(64 * 512, 128 * 512, PAGE_SIZE), + src_file_("copy_src", 32), + dest_file_("copy_dest", 32), + engine_(new aio_engine(MAX_IO)) { + } + + // in sectors + static uint64_t meg(unsigned n) { + return 2 * 1024 * n; + } + + mempool pool_; + temp_file src_file_; + temp_file dest_file_; + unique_ptr engine_; + }; +} + +//---------------------------------------------------------------- + +TEST_F(IOEngineTests, empty_test) +{ +} + +TEST_F(IOEngineTests, open_and_close) +{ + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + auto dest_handle = engine_->open_file(dest_file_.get_path(), io_engine::READ_WRITE); + ASSERT_TRUE(src_handle != dest_handle); + engine_->close_file(src_handle); + engine_->close_file(dest_handle); +} + +TEST_F(IOEngineTests, you_can_read_a_read_only_handle) +{ + unsigned nr_sectors = 8; + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + void *data = pool_.alloc(); + bool r = engine_->issue_io(src_handle, + io_engine::READ, + 0, nr_sectors, + data, + 123); + ASSERT_TRUE(r); + auto wr = engine_->wait(); + ASSERT_TRUE(wr.first); + ASSERT_TRUE(wr.second == 123); + + engine_->close_file(src_handle); + pool_.free(data); +} + + +TEST_F(IOEngineTests, you_cannot_write_to_a_read_only_handle) +{ + unsigned nr_sectors = 8; + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + void *data = pool_.alloc(); + bool r = engine_->issue_io(src_handle, + io_engine::WRITE, + 0, nr_sectors, + data, + 0); + ASSERT_FALSE(r); + engine_->close_file(src_handle); + pool_.free(data); +} + +TEST_F(IOEngineTests, you_can_write_to_a_read_write_handle) +{ + unsigned nr_sectors = 8; + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + void *data = pool_.alloc(); + bool r = engine_->issue_io(src_handle, + io_engine::READ, + 0, nr_sectors, + data, + 123); + ASSERT_TRUE(r); + auto wr = engine_->wait(); + ASSERT_TRUE(wr.first); + ASSERT_TRUE(wr.second == 123); + + engine_->close_file(src_handle); + pool_.free(data); +} + +TEST_F(IOEngineTests, final_block_read_succeeds) +{ + unsigned nr_sectors = 8; + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + void *data = pool_.alloc(); + bool r = engine_->issue_io(src_handle, + io_engine::READ, + meg(32) - nr_sectors, meg(32), + data, + 123); + ASSERT_TRUE(r); + auto wr = engine_->wait(); + ASSERT_TRUE(wr.first); + + engine_->close_file(src_handle); + pool_.free(data); + +} + +TEST_F(IOEngineTests, out_of_bounds_read_fails) +{ + unsigned nr_sectors = 8; + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + void *data = pool_.alloc(); + bool r = engine_->issue_io(src_handle, + io_engine::READ, + meg(32), meg(32) + nr_sectors, + data, + 123); + ASSERT_TRUE(r); + auto wr = engine_->wait(); + ASSERT_FALSE(wr.first); + + engine_->close_file(src_handle); + pool_.free(data); + +} + +TEST_F(IOEngineTests, out_of_bounds_write_succeeds) +{ + unsigned nr_sectors = 8; + auto handle = engine_->open_file(dest_file_.get_path(), io_engine::READ_WRITE); + void *data = pool_.alloc(); + bool r = engine_->issue_io(handle, + io_engine::WRITE, + meg(32), meg(32) + nr_sectors, + data, + 123); + ASSERT_TRUE(r); + auto wr = engine_->wait(); + ASSERT_TRUE(wr.first); + + engine_->close_file(handle); + pool_.free(data); + +} + +//---------------------------------------------------------------- diff --git a/unit-tests/mem_pool_t.cc b/unit-tests/mem_pool_t.cc index 24fb34c..43e05ec 100644 --- a/unit-tests/mem_pool_t.cc +++ b/unit-tests/mem_pool_t.cc @@ -93,4 +93,20 @@ TEST_F(MempoolTests, exhaust_pool) ASSERT_EQ(md, nullptr); } +// Use valgrind +TEST_F(MempoolTests, data_can_be_written) +{ + mempool mp(512, 100 * 512, 512); + + for (unsigned i = 0; i < 100; i++) { + auto md = mp.alloc(); + ASSERT_NE(md, nullptr); + + memset(md, 0, 512); + } + + auto md = mp.alloc(); + ASSERT_EQ(md, nullptr); +} + //---------------------------------------------------------------- diff --git a/unit-tests/test_utils.cc b/unit-tests/test_utils.cc index 5a9cc96..85876bd 100644 --- a/unit-tests/test_utils.cc +++ b/unit-tests/test_utils.cc @@ -3,6 +3,8 @@ #include "persistent-data/space-maps/core.h" using namespace persistent_data; +using namespace std; +using namespace test; //---------------------------------------------------------------- @@ -21,3 +23,35 @@ test::open_temporary_tm(block_manager<>::ptr bm) } //---------------------------------------------------------------- + +temp_file::temp_file(string const &name_base, unsigned meg_size) + : path_(gen_path(name_base)) +{ + int fd = ::open(path_.c_str(), O_CREAT | O_RDWR, 0666); + if (fd < 0) + throw runtime_error("couldn't open file"); + + if (::fallocate(fd, 0, 0, 1024 * 1024 * meg_size)) + throw runtime_error("couldn't fallocate"); + + ::close(fd); +} + +temp_file::~temp_file() +{ +// ::unlink(path_.c_str()); +} + +string const & +temp_file::get_path() const +{ + return path_; +} + +string +temp_file::gen_path(string const &base) +{ + return string("./") + base + string(".tmp"); +} + +//---------------------------------------------------------------- diff --git a/unit-tests/test_utils.h b/unit-tests/test_utils.h index bdd63e5..e979d07 100644 --- a/unit-tests/test_utils.h +++ b/unit-tests/test_utils.h @@ -114,6 +114,18 @@ namespace test { std::unique_ptr dir_; }; + class temp_file { + public: + temp_file(std::string const &name_base, unsigned meg_size); + ~temp_file(); + std::string const &get_path() const; + + private: + static string gen_path(string const &base); + + string path_; + }; + //-------------------------------- template From df9f4d479d67348748eb80b376256d931481869d Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 7 Jun 2016 11:15:35 +0100 Subject: [PATCH 101/118] update get-gmock script. You now need git, since they haven't released for a while. --- get-gmock.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/get-gmock.sh b/get-gmock.sh index 5335350..2e5d2ae 100755 --- a/get-gmock.sh +++ b/get-gmock.sh @@ -1,6 +1,3 @@ #!/bin/sh -e -wget https://googlemock.googlecode.com/files/gmock-1.6.0.zip -unzip gmock-1.6.0.zip -cd gmock-1.6.0 -./configure +git clone https://github.com/google/googletest From 07f44e9c77892eea25cd22065241d170fb4f58df Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 7 Jun 2016 13:45:27 +0100 Subject: [PATCH 102/118] [io_engine] Add exclusive flag to io_engine --- block-cache/io_engine.cc | 4 +++- block-cache/io_engine.h | 10 +++++++--- unit-tests/copier_t.cc | 6 +++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/block-cache/io_engine.cc b/block-cache/io_engine.cc index 7837c33..a4211cb 100644 --- a/block-cache/io_engine.cc +++ b/block-cache/io_engine.cc @@ -75,9 +75,11 @@ aio_engine::~aio_engine() } aio_engine::handle -aio_engine::open_file(std::string const &path, mode m) +aio_engine::open_file(std::string const &path, mode m, sharing s) { int flags = (m == READ_ONLY) ? O_RDONLY : O_RDWR; + if (s == EXCLUSIVE) + flags |= O_EXCL; int fd = ::open(path.c_str(), O_DIRECT | flags); if (fd < 0) { ostringstream out; diff --git a/block-cache/io_engine.h b/block-cache/io_engine.h index 55b556d..f9e4c9c 100644 --- a/block-cache/io_engine.h +++ b/block-cache/io_engine.h @@ -27,12 +27,17 @@ namespace bcache { WRITE }; + enum sharing { + EXCLUSIVE, + SHARED + }; + io_engine() {} virtual ~io_engine() {} using handle = unsigned; - virtual handle open_file(std::string const &path, mode m) = 0; + virtual handle open_file(std::string const &path, mode m, sharing s = EXCLUSIVE) = 0; virtual void close_file(handle h) = 0; // returns false if there are insufficient resources to @@ -79,8 +84,7 @@ namespace bcache { using handle = unsigned; - // FIXME: open exclusive? - virtual handle open_file(std::string const &path, mode m); + virtual handle open_file(std::string const &path, mode m, sharing s = EXCLUSIVE); virtual void close_file(handle h); // Returns false if queueing the io failed diff --git a/unit-tests/copier_t.cc b/unit-tests/copier_t.cc index e95819f..0185d9e 100644 --- a/unit-tests/copier_t.cc +++ b/unit-tests/copier_t.cc @@ -33,7 +33,7 @@ using namespace testing; namespace { class io_engine_mock : public io_engine { public: - MOCK_METHOD2(open_file, handle(string const &, mode)); + MOCK_METHOD3(open_file, handle(string const &, mode, sharing)); MOCK_METHOD1(close_file, void(handle)); MOCK_METHOD6(issue_io, bool(handle, dir, sector_t, sector_t, void *, unsigned)); @@ -51,9 +51,9 @@ namespace { } unique_ptr make_copier() { - EXPECT_CALL(engine_, open_file(src_file_, io_engine::READ_ONLY)). + EXPECT_CALL(engine_, open_file(src_file_, io_engine::READ_ONLY, io_engine::EXCLUSIVE)). WillOnce(Return(SRC_HANDLE)); - EXPECT_CALL(engine_, open_file(dest_file_, io_engine::READ_WRITE)). + EXPECT_CALL(engine_, open_file(dest_file_, io_engine::READ_WRITE, io_engine::EXCLUSIVE)). WillOnce(Return(DEST_HANDLE)); EXPECT_CALL(engine_, close_file(SRC_HANDLE)).Times(1); From e5f969817e6c11f3366399f679279d3c17b31623 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 14 Jun 2016 14:27:22 +0100 Subject: [PATCH 103/118] [block cache] improve exception message --- block-cache/block_cache.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/block-cache/block_cache.cc b/block-cache/block_cache.cc index 16e355c..98460d9 100644 --- a/block-cache/block_cache.cc +++ b/block-cache/block_cache.cc @@ -525,8 +525,11 @@ block_cache::get(block_address index, unsigned flags, validator::ptr v) block *b = lookup_or_read_block(index, flags, v); if (b) { - if (b->ref_count_ && flags & (GF_DIRTY | GF_ZERO)) - throw std::runtime_error("attempt to write lock block concurrently"); + if (b->ref_count_ && (flags & (GF_DIRTY | GF_ZERO))) { + std::ostringstream out; + out << "attempt to write lock block " << index << " concurrently"; + throw std::runtime_error(out.str()); + } // FIXME: this gets called even for new blocks hit(*b); From 0f778a0a38e6ef81f76629fec2428745fccbf33a Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 14 Jun 2016 16:26:37 +0100 Subject: [PATCH 104/118] [block-cache] FIx some bugs in the io engine --- block-cache/io_engine.cc | 61 +++++++++++++++++++++++++++++----------- block-cache/io_engine.h | 22 ++++++++++----- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/block-cache/io_engine.cc b/block-cache/io_engine.cc index a4211cb..2fe4082 100644 --- a/block-cache/io_engine.cc +++ b/block-cache/io_engine.cc @@ -14,13 +14,6 @@ using namespace std; //---------------------------------------------------------------- -namespace { - unsigned const SECTOR_SHIFT = 9; - unsigned const PAGE_SIZE = 4096; -} - -//---------------------------------------------------------------- - control_block_set::control_block_set(unsigned nr) : cbs_(nr) { @@ -77,7 +70,7 @@ aio_engine::~aio_engine() aio_engine::handle aio_engine::open_file(std::string const &path, mode m, sharing s) { - int flags = (m == READ_ONLY) ? O_RDONLY : O_RDWR; + int flags = (m == M_READ_ONLY) ? O_RDONLY : O_RDWR; if (s == EXCLUSIVE) flags |= O_EXCL; int fd = ::open(path.c_str(), O_DIRECT | flags); @@ -126,45 +119,81 @@ aio_engine::issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsign cb->u.c.buf = data; cb->u.c.offset = b << SECTOR_SHIFT; cb->u.c.nbytes = (e - b) << SECTOR_SHIFT; - cb->aio_lio_opcode = (d == READ) ? IO_CMD_PREAD : IO_CMD_PWRITE; + cb->aio_lio_opcode = (d == D_READ) ? IO_CMD_PREAD : IO_CMD_PWRITE; int r = io_submit(aio_context_, 1, &cb); return r == 1; } -std::pair +optional aio_engine::wait() +{ + return wait_(NULL); +} + +optional +aio_engine::wait(unsigned µsec) +{ + timespec start = micro_to_ts(microsec); + timespec stop = start; + auto r = wait_(&stop); + microsec = ts_to_micro(stop) - microsec; + return r; +} + +boost::optional +aio_engine::wait_(timespec *ts) { int r; struct io_event event; memset(&event, 0, sizeof(event)); - - r = io_getevents(aio_context_, 1, 1, &event, NULL); + r = io_getevents(aio_context_, 1, 1, &event, ts); if (r < 0) { std::ostringstream out; out << "io_getevents failed: " << r; throw std::runtime_error(out.str()); } + if (r == 0) { + return optional(); + } + iocb *cb = reinterpret_cast(event.obj); unsigned context = cbs_.context(cb); if (event.res == cb->u.c.nbytes) { cbs_.free(cb); - return make_pair(true, context); + return optional(make_pair(true, context)); } else if (static_cast(event.res) < 0) { cbs_.free(cb); - return make_pair(false, context); + return optional(make_pair(false, context)); } else { cbs_.free(cb); - return make_pair(false, context); + return optional(make_pair(false, context)); } // shouldn't get here - return make_pair(false, 0); + return optional(make_pair(false, 0)); +} + +struct timespec +aio_engine::micro_to_ts(unsigned micro) +{ + timespec ts; + ts.tv_sec = micro / 1000000u; + ts.tv_nsec = (micro % 1000000) * 1000; + return ts; +} + +unsigned +aio_engine::ts_to_micro(timespec const &ts) +{ + unsigned micro = ts.tv_sec * 1000000; + micro += ts.tv_nsec / 1000; + return micro; } //---------------------------------------------------------------- diff --git a/block-cache/io_engine.h b/block-cache/io_engine.h index f9e4c9c..fd902c1 100644 --- a/block-cache/io_engine.h +++ b/block-cache/io_engine.h @@ -14,17 +14,20 @@ namespace bcache { using sector_t = uint64_t; + unsigned const SECTOR_SHIFT = 9; + unsigned const PAGE_SIZE = 4096; + // Virtual base class to aid unit testing class io_engine { public: enum mode { - READ_ONLY, - READ_WRITE + M_READ_ONLY, + M_READ_WRITE }; enum dir { - READ, - WRITE + D_READ, + D_WRITE }; enum sharing { @@ -46,7 +49,8 @@ namespace bcache { // returns (success, context) using wait_result = std::pair; - virtual wait_result wait() = 0; + virtual boost::optional wait() = 0; + virtual boost::optional wait(unsigned µsec) = 0; private: io_engine(io_engine const &) = delete; @@ -90,10 +94,14 @@ namespace bcache { // Returns false if queueing the io failed virtual bool issue_io(handle h, dir d, sector_t b, sector_t e, void *data, unsigned context); - // returns (success, context) - virtual wait_result wait(); + virtual boost::optional wait(); + virtual boost::optional wait(unsigned µsec); private: + static struct timespec micro_to_ts(unsigned micro); + static unsigned ts_to_micro(timespec const &ts); + boost::optional wait_(timespec *ts); + std::list descriptors_; io_context_t aio_context_; From a124b7ce2639391607fac832edf418be4f43cb68 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 14 Jun 2016 16:27:17 +0100 Subject: [PATCH 105/118] [block-cache] Fix some bugs in the copier --- block-cache/copier.cc | 87 ++++++++++++++++++++++++++++++++------- block-cache/copier.h | 9 ++++ unit-tests/copier_t.cc | 87 +++++++++++++++++++++++++++++---------- unit-tests/io_engine_t.cc | 63 ++++++++++++++++++---------- 4 files changed, 188 insertions(+), 58 deletions(-) diff --git a/block-cache/copier.cc b/block-cache/copier.cc index 9910b58..b5c21ef 100644 --- a/block-cache/copier.cc +++ b/block-cache/copier.cc @@ -11,12 +11,12 @@ using namespace std; copier::copier(io_engine &engine, string const &src, string const &dest, sector_t block_size, size_t mem) - : pool_(block_size * 512, mem), + : pool_(block_size * 512, mem, PAGE_SIZE), block_size_(block_size), nr_blocks_(mem / block_size), engine_(engine), - src_handle_(engine_.open_file(src, io_engine::READ_ONLY)), - dest_handle_(engine_.open_file(dest, io_engine::READ_WRITE)), + src_handle_(engine_.open_file(src, io_engine::M_READ_ONLY)), + dest_handle_(engine_.open_file(dest, io_engine::M_READ_WRITE)), genkey_count_(0) { } @@ -30,14 +30,14 @@ copier::~copier() void copier::issue(copy_op const &op) { - auto data = pool_.alloc(); - if (!data) { - wait_(); - data = pool_.alloc(); + void *data; - if (!data) - // Shouldn't get here - throw runtime_error("couldn't allocate buffer"); + while (!(data = pool_.alloc())) { + wait_(); + + // data may still not be present because the wait_ could + // have completed a read and issued the corresponding + // write. } copy_job job(op, data); @@ -45,7 +45,7 @@ copier::issue(copy_op const &op) unsigned key = genkey(); // used as context for the io_engine auto r = engine_.issue_io(src_handle_, - io_engine::READ, + io_engine::D_READ, to_sector(op.src_b), to_sector(op.src_e), data, @@ -53,6 +53,7 @@ copier::issue(copy_op const &op) if (r) jobs_.insert(make_pair(key, job)); + else complete(job); } @@ -66,23 +67,74 @@ copier::nr_pending() const boost::optional copier::wait() { - while (!jobs_.empty() && complete_.empty()) + if (complete_.empty()) wait_(); + return wait_complete(); +} + +boost::optional +copier::wait(unsigned µ) +{ if (complete_.empty()) + wait_(micro); + return wait_complete(); +} + +bool +copier::pending() const +{ + return !jobs_.empty(); +} + +boost::optional +copier::wait_complete() +{ + if (complete_.empty()) { return optional(); - else { + } else { auto op = complete_.front(); complete_.pop_front(); return optional(op); } } +void +copier::wait_(unsigned µ) +{ + optional mp; + + if (!pending()) + return; + + + bool completed = false; + while (pending() && !completed) { + mp = engine_.wait(micro); + if (mp) + completed = wait_successful(*mp); + + if (!micro) + break; + } +} + void copier::wait_() { - auto p = engine_.wait(); + bool completed = false; + + while (pending() && !completed) { + auto mp = engine_.wait(); + if (mp) + completed = wait_successful(*mp); + } +} + +bool +copier::wait_successful(io_engine::wait_result const &p) +{ auto it = jobs_.find(p.second); if (it == jobs_.end()) throw runtime_error("Internal error. Lost track of copy job."); @@ -92,26 +144,29 @@ copier::wait_() // IO was unsuccessful complete(j); jobs_.erase(it); - return; + return true; } // IO was successful if (!j.op.read_complete) { j.op.read_complete = true; if (!engine_.issue_io(dest_handle_, - io_engine::WRITE, + io_engine::D_WRITE, to_sector(j.op.dest_b), to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)), j.data, it->first)) { complete(j); jobs_.erase(it); + return true; } + return false; } else { j.op.write_complete = true; complete(j); jobs_.erase(it); + return true; } } diff --git a/block-cache/copier.h b/block-cache/copier.h index 5eb2a51..bd49966 100644 --- a/block-cache/copier.h +++ b/block-cache/copier.h @@ -32,6 +32,10 @@ namespace bcache { write_complete(false) { } + bool operator <(copy_op const &rhs) const { + return dest_b < rhs.dest_b; + } + bool success() const { return read_complete && write_complete; } @@ -69,8 +73,13 @@ namespace bcache { unsigned nr_pending() const; boost::optional wait(); + boost::optional wait(unsigned µ); private: + bool pending() const; + bool wait_successful(io_engine::wait_result const &p); + boost::optional wait_complete(); + void wait_(unsigned µ); void wait_(); void complete(copy_job const &j); diff --git a/unit-tests/copier_t.cc b/unit-tests/copier_t.cc index 0185d9e..1621c4c 100644 --- a/unit-tests/copier_t.cc +++ b/unit-tests/copier_t.cc @@ -20,7 +20,6 @@ #include "block-cache/copier.h" #include "test_utils.h" - #include using namespace boost; @@ -31,18 +30,32 @@ using namespace testing; //---------------------------------------------------------------- namespace { + unsigned const BLOCK_SIZE = 64u; + using wait_result = io_engine::wait_result; + + ostream &operator <<(ostream &out, wait_result const &wr) { + out << "wait_result[" << wr.first << ", " << wr.second << "]"; + return out; + } + + ostream &operator <<(ostream &out, optional const &mwr) { + if (mwr) { + out << "Just[wait_result[" << mwr->first << ", " << mwr->second << "]]"; + } else + out << "Nothing"; + return out; + } + class io_engine_mock : public io_engine { public: MOCK_METHOD3(open_file, handle(string const &, mode, sharing)); MOCK_METHOD1(close_file, void(handle)); MOCK_METHOD6(issue_io, bool(handle, dir, sector_t, sector_t, void *, unsigned)); - MOCK_METHOD0(wait, wait_result()); + MOCK_METHOD0(wait, optional()); + MOCK_METHOD1(wait, optional(unsigned &)); }; - unsigned const BLOCK_SIZE = 64u; - using wait_result = io_engine::wait_result; - class CopierTests : public Test { public: CopierTests() @@ -51,9 +64,9 @@ namespace { } unique_ptr make_copier() { - EXPECT_CALL(engine_, open_file(src_file_, io_engine::READ_ONLY, io_engine::EXCLUSIVE)). + EXPECT_CALL(engine_, open_file(src_file_, io_engine::M_READ_ONLY, io_engine::EXCLUSIVE)). WillOnce(Return(SRC_HANDLE)); - EXPECT_CALL(engine_, open_file(dest_file_, io_engine::READ_WRITE, io_engine::EXCLUSIVE)). + EXPECT_CALL(engine_, open_file(dest_file_, io_engine::M_READ_WRITE, io_engine::EXCLUSIVE)). WillOnce(Return(DEST_HANDLE)); EXPECT_CALL(engine_, close_file(SRC_HANDLE)).Times(1); @@ -64,11 +77,15 @@ namespace { BLOCK_SIZE, 1 * 1024 * 1024)); } + static optional make_wr(bool success, unsigned context) { + return optional(wait_result(success, context)); + } + void issue_successful_op(copier &c, copy_op &op, unsigned context) { InSequence dummy; unsigned nr_pending = c.nr_pending(); - EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, op.src_b * BLOCK_SIZE, op.src_e * BLOCK_SIZE, _, context)). WillOnce(Return(true)); @@ -77,15 +94,15 @@ namespace { ASSERT_TRUE(c.nr_pending() == nr_pending + 1); EXPECT_CALL(engine_, wait()). - WillOnce(Return(wait_result(true, context))); + WillOnce(Return(make_wr(true, context))); - EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::WRITE, + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE, op.dest_b * BLOCK_SIZE, (op.dest_b + (op.src_e - op.src_b)) * BLOCK_SIZE, _, context)). WillOnce(Return(true)); EXPECT_CALL(engine_, wait()). - WillOnce(Return(wait_result(true, context))); + WillOnce(Return(make_wr(true, context))); auto mop = c.wait(); ASSERT_EQ(c.nr_pending(), nr_pending); @@ -124,7 +141,7 @@ TEST_F(CopierTests, unsuccessful_issue_read) auto c = make_copier(); InSequence dummy; - EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, 0, BLOCK_SIZE, _, 0)). WillOnce(Return(false)); c->issue(op1); @@ -141,13 +158,13 @@ TEST_F(CopierTests, unsuccessful_read) auto c = make_copier(); InSequence dummy; - EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, 0, BLOCK_SIZE, _, 0)). WillOnce(Return(true)); c->issue(op1); ASSERT_EQ(c->nr_pending(), 1u); EXPECT_CALL(engine_, wait()). - WillOnce(Return(wait_result(false, 0u))); + WillOnce(Return(make_wr(false, 0u))); ASSERT_EQ(c->nr_pending(), 1u); auto mop = c->wait(); @@ -161,17 +178,17 @@ TEST_F(CopierTests, unsuccessful_issue_write) auto c = make_copier(); InSequence dummy; - EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, 0, BLOCK_SIZE, _, 0)). WillOnce(Return(true)); c->issue(op1); ASSERT_EQ(c->nr_pending(), 1u); EXPECT_CALL(engine_, wait()). - WillOnce(Return(wait_result(true, 0u))); + WillOnce(Return(make_wr(true, 0u))); ASSERT_EQ(c->nr_pending(), 1u); - EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::WRITE, 0, BLOCK_SIZE, _, 0)). + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE, 0, BLOCK_SIZE, _, 0)). WillOnce(Return(false)); auto mop = c->wait(); @@ -187,19 +204,19 @@ TEST_F(CopierTests, unsuccessful_write) auto c = make_copier(); InSequence dummy; - EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::READ, 0, BLOCK_SIZE, _, 0)). + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, 0, BLOCK_SIZE, _, 0)). WillOnce(Return(true)); c->issue(op1); ASSERT_EQ(c->nr_pending(), 1u); EXPECT_CALL(engine_, wait()). - WillOnce(Return(wait_result(true, 0u))); + WillOnce(Return(make_wr(true, 0u))); - EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::WRITE, 0, BLOCK_SIZE, _, 0)). + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE, 0, BLOCK_SIZE, _, 0)). WillOnce(Return(true)); EXPECT_CALL(engine_, wait()). - WillOnce(Return(wait_result(false, 0u))); + WillOnce(Return(make_wr(false, 0u))); auto mop = c->wait(); ASSERT_EQ(c->nr_pending(), 0u); @@ -225,4 +242,32 @@ TEST_F(CopierTests, copy_different_blocks) } } +TEST_F(CopierTests, wait_can_timeout) +{ + copy_op op1(0, 1, 0); + auto c = make_copier(); + + InSequence dummy; + EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(true)); + c->issue(op1); + + ASSERT_EQ(c->nr_pending(), 1u); + + unsigned micro = 10000; + EXPECT_CALL(engine_, wait(micro)). + WillOnce(Return(make_wr(true, 0u))); + ASSERT_EQ(c->nr_pending(), 1u); + + EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE, 0, BLOCK_SIZE, _, 0)). + WillOnce(Return(true)); + + EXPECT_CALL(engine_, wait(micro)). + WillOnce(DoAll(SetArgReferee<0>(0u), Return(optional()))); + + auto mop = c->wait(micro); + ASSERT_FALSE(mop); + ASSERT_EQ(c->nr_pending(), 1u); +} + //---------------------------------------------------------------- diff --git a/unit-tests/io_engine_t.cc b/unit-tests/io_engine_t.cc index f1b3a9e..2790eb6 100644 --- a/unit-tests/io_engine_t.cc +++ b/unit-tests/io_engine_t.cc @@ -64,8 +64,8 @@ TEST_F(IOEngineTests, empty_test) TEST_F(IOEngineTests, open_and_close) { - auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); - auto dest_handle = engine_->open_file(dest_file_.get_path(), io_engine::READ_WRITE); + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); + auto dest_handle = engine_->open_file(dest_file_.get_path(), io_engine::M_READ_WRITE); ASSERT_TRUE(src_handle != dest_handle); engine_->close_file(src_handle); engine_->close_file(dest_handle); @@ -74,17 +74,17 @@ TEST_F(IOEngineTests, open_and_close) TEST_F(IOEngineTests, you_can_read_a_read_only_handle) { unsigned nr_sectors = 8; - auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); void *data = pool_.alloc(); bool r = engine_->issue_io(src_handle, - io_engine::READ, + io_engine::D_READ, 0, nr_sectors, data, 123); ASSERT_TRUE(r); auto wr = engine_->wait(); - ASSERT_TRUE(wr.first); - ASSERT_TRUE(wr.second == 123); + ASSERT_TRUE(wr->first); + ASSERT_TRUE(wr->second == 123); engine_->close_file(src_handle); pool_.free(data); @@ -94,10 +94,10 @@ TEST_F(IOEngineTests, you_can_read_a_read_only_handle) TEST_F(IOEngineTests, you_cannot_write_to_a_read_only_handle) { unsigned nr_sectors = 8; - auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); void *data = pool_.alloc(); bool r = engine_->issue_io(src_handle, - io_engine::WRITE, + io_engine::D_WRITE, 0, nr_sectors, data, 0); @@ -109,17 +109,17 @@ TEST_F(IOEngineTests, you_cannot_write_to_a_read_only_handle) TEST_F(IOEngineTests, you_can_write_to_a_read_write_handle) { unsigned nr_sectors = 8; - auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); void *data = pool_.alloc(); bool r = engine_->issue_io(src_handle, - io_engine::READ, + io_engine::D_READ, 0, nr_sectors, data, 123); ASSERT_TRUE(r); auto wr = engine_->wait(); - ASSERT_TRUE(wr.first); - ASSERT_TRUE(wr.second == 123); + ASSERT_TRUE(wr->first); + ASSERT_TRUE(wr->second == 123); engine_->close_file(src_handle); pool_.free(data); @@ -128,16 +128,16 @@ TEST_F(IOEngineTests, you_can_write_to_a_read_write_handle) TEST_F(IOEngineTests, final_block_read_succeeds) { unsigned nr_sectors = 8; - auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); void *data = pool_.alloc(); bool r = engine_->issue_io(src_handle, - io_engine::READ, + io_engine::D_READ, meg(32) - nr_sectors, meg(32), data, 123); ASSERT_TRUE(r); auto wr = engine_->wait(); - ASSERT_TRUE(wr.first); + ASSERT_TRUE(wr->first); engine_->close_file(src_handle); pool_.free(data); @@ -147,16 +147,16 @@ TEST_F(IOEngineTests, final_block_read_succeeds) TEST_F(IOEngineTests, out_of_bounds_read_fails) { unsigned nr_sectors = 8; - auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::READ_ONLY); + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); void *data = pool_.alloc(); bool r = engine_->issue_io(src_handle, - io_engine::READ, + io_engine::D_READ, meg(32), meg(32) + nr_sectors, data, 123); ASSERT_TRUE(r); auto wr = engine_->wait(); - ASSERT_FALSE(wr.first); + ASSERT_FALSE(wr->first); engine_->close_file(src_handle); pool_.free(data); @@ -166,20 +166,41 @@ TEST_F(IOEngineTests, out_of_bounds_read_fails) TEST_F(IOEngineTests, out_of_bounds_write_succeeds) { unsigned nr_sectors = 8; - auto handle = engine_->open_file(dest_file_.get_path(), io_engine::READ_WRITE); + auto handle = engine_->open_file(dest_file_.get_path(), io_engine::M_READ_WRITE); void *data = pool_.alloc(); bool r = engine_->issue_io(handle, - io_engine::WRITE, + io_engine::D_WRITE, meg(32), meg(32) + nr_sectors, data, 123); ASSERT_TRUE(r); auto wr = engine_->wait(); - ASSERT_TRUE(wr.first); + ASSERT_TRUE(wr->first); engine_->close_file(handle); pool_.free(data); } +TEST_F(IOEngineTests, succeed_with_timeout) +{ + unsigned nr_sectors = 8; + auto src_handle = engine_->open_file(src_file_.get_path(), io_engine::M_READ_ONLY); + void *data = pool_.alloc(); + bool r = engine_->issue_io(src_handle, + io_engine::D_READ, + 0, nr_sectors, + data, + 123); + ASSERT_TRUE(r); + unsigned micro = 10; + auto wr = engine_->wait(micro); + ASSERT_TRUE(wr->first); + ASSERT_TRUE(wr->second == 123); + + engine_->close_file(src_handle); + pool_.free(data); +} + + //---------------------------------------------------------------- From b77ba14a2a6d52c1675e1c034a57d93f887216db Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 14 Jun 2016 16:28:14 +0100 Subject: [PATCH 106/118] [block] Use the definition of SECTOR_SHIFT from block-cache --- persistent-data/block.tcc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/persistent-data/block.tcc b/persistent-data/block.tcc index 0824bf3..02a4117 100644 --- a/persistent-data/block.tcc +++ b/persistent-data/block.tcc @@ -19,6 +19,7 @@ #include "block.h" #include "base/error_string.h" +#include "block-cache/io_engine.h" #include #include @@ -38,8 +39,6 @@ namespace { using namespace std; int const DEFAULT_MODE = 0666; - unsigned const SECTOR_SHIFT = 9; - int const OPEN_FLAGS = O_DIRECT; // FIXME: introduce a new exception for this, or at least lift this From 8989bb0f326d56e0f9c32d7a37ad9004530fc41f Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 14 Jun 2016 16:29:37 +0100 Subject: [PATCH 107/118] [caching] cache_writeback --- caching/cache_writeback.cc | 256 +++++++++++++++++++++++++++++++++---- 1 file changed, 234 insertions(+), 22 deletions(-) diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc index f7b9a28..7dbf4c3 100644 --- a/caching/cache_writeback.cc +++ b/caching/cache_writeback.cc @@ -1,3 +1,4 @@ +#include "base/progress_monitor.h" #include "persistent-data/file_utils.h" #include "block-cache/copier.h" #include "caching/commands.h" @@ -18,28 +19,99 @@ using namespace std; //---------------------------------------------------------------- namespace { + + template T safe_div(T const n, T const d, T const def) { + return (d == T()) ? def : (n / d); + } + + //-------------------------------- + struct flags { flags() - : cache_size(1024 * 1024 * 128) { + : cache_size(4 * 1024 * 1024), + sort_buffers(16 * 1024), + list_failed_blocks(false), + update_metadata(true) { + } + + // The sort buffers have a dramatic effect on the + // performance. We give up 10% of the general buffer space + // for them. + void calc_sort_buffer_size() { + size_t sbs = cache_size / 10; + cache_size = cache_size - sbs; + + sort_buffers = sbs / sizeof(copy_op); } using maybe_string = boost::optional; size_t cache_size; + unsigned sort_buffers; maybe_string metadata_dev; maybe_string origin_dev; maybe_string fast_dev; + bool list_failed_blocks; + bool update_metadata; + }; + + //-------------------------------- + + class copy_batch { + public: + copy_batch(unsigned nr) + : max_(nr), + count_(0), + ops_(nr) { + } + + bool space() const { + return count_ < max_; + } + + void push_op(copy_op const &op) { + if (!space()) + throw runtime_error("copy_batch out of space"); + + ops_[count_++] = op; + } + + void reset() { + count_ = 0; + } + + vector::iterator begin() { + return ops_.begin(); + } + + vector::iterator end() { + return ops_.begin() + count_; + } + + private: + unsigned max_; + unsigned count_; + vector ops_; }; class copy_visitor : public mapping_visitor { public: - copy_visitor(copier &c, bool only_dirty) + copy_visitor(copier &c, unsigned sort_buffer, bool only_dirty, + bool list_failed_blocks, + progress_monitor &monitor, unsigned cache_blocks) : copier_(c), block_size_(c.get_block_size()), - only_dirty_(only_dirty) { + only_dirty_(only_dirty), + list_failed_blocks_(list_failed_blocks), + batch_(sort_buffer), + monitor_(monitor), + cache_blocks_(cache_blocks) { } virtual void visit(block_address cblock, mapping const &m) { + stats_.blocks_scanned = cblock; + update_monitor(); + if (!(m.flags_ & M_VALID)) return; @@ -52,41 +124,122 @@ namespace { cop.dest_b = m.oblock_; // blocks - copier_.issue(cop); + stats_.blocks_needed++; + batch_.push_op(cop); + if (!batch_.space()) + issue(); + } - stats_.blocks_issued++; + void issue() { + auto compare_dest = [](copy_op const &lhs, copy_op const &rhs) { + return lhs.dest_b < rhs.dest_b; + }; + sort(batch_.begin(), batch_.end(), compare_dest); -#if 0 - if (sectors_copied < block_size_) { - stats_.blocks_failed++; - stats_.sectors_failed += block_size_ - sectors_copied; + auto e = batch_.end(); + for (auto it = batch_.begin(); it != e; ++it) { + copier_.issue(*it); + stats_.blocks_issued++; + update_monitor(); + + check_for_completed_copies(); } -#endif + check_for_completed_copies(); + + batch_.reset(); + } + + void check_for_completed_copies(bool block = false) { + optional mop; + + do { + if (block) + mop = copier_.wait(); + + else { + unsigned micro = 0; + mop = copier_.wait(micro); + } + + if (mop) { + inc_completed(*mop); + if (!mop->success()) { + failed_blocks_.insert(*mop); + failed_cblocks_.insert(mop->src_b); + } + } + + } while (mop); + } + + void complete() { + issue(); + + while (copier_.nr_pending()) + check_for_completed_copies(true); + + monitor_.update_percent(100); + cerr << "\n"; + } + + void inc_completed(copy_op const &op) { + stats_.blocks_completed++; + update_monitor(); + } + + void update_monitor() { + static unsigned call_count = 0; + if (call_count++ % 128) + return; + + uint64_t scanned = stats_.blocks_scanned * 100 / cache_blocks_; + uint64_t copied = safe_div(stats_.blocks_completed * 100, + stats_.blocks_needed, 100ull); + uint64_t percent = min(scanned, copied); + monitor_.update_percent(percent); } struct copy_stats { copy_stats() - : blocks_issued(0), - blocks_failed(0), - sectors_failed(0) { + : blocks_scanned(0), + blocks_needed(0), + blocks_issued(0), + blocks_completed(0), + blocks_failed(0) { } + block_address blocks_scanned; + block_address blocks_needed; block_address blocks_issued; + block_address blocks_completed; block_address blocks_failed; - block_address sectors_failed; }; copy_stats const &get_stats() const { return stats_; } + set failed_writebacks() const { + return failed_cblocks_; + } + private: copier &copier_; unsigned block_size_; bool only_dirty_; + bool list_failed_blocks_; + copy_stats stats_; + copy_batch batch_; + progress_monitor &monitor_; + unsigned cache_blocks_; + + set failed_blocks_; + set failed_cblocks_; }; + //-------------------------------- + using namespace mapping_array_damage; class ignore_damage_visitor : public damage_visitor { @@ -117,25 +270,65 @@ namespace { return md.sb_.flags.get_flag(superblock_flags::CLEAN_SHUTDOWN); } + void update_metadata(metadata &md, set const &failed_writebacks) { + cout << "Updating metadata ... "; + + cout.flush(); + + auto &mappings = md.mappings_; + for (block_address cblock = 0; cblock < mappings->get_nr_entries(); cblock++) { + auto m = mappings->get(cblock); + if (!(m.flags_ & M_VALID)) + continue; + + if (!(m.flags_ & M_DIRTY)) + continue; + + if (failed_writebacks.count(cblock)) + continue; + + m.flags_ &= ~M_DIRTY; + cerr << "clearing dirty flag for block " << cblock << "\n"; + mappings->set(cblock, m); + } + md.commit(true); + cout << "done\n"; + cout.flush(); + } + int writeback_(flags const &f) { - block_manager<>::ptr bm = open_bm(*f.metadata_dev, block_manager<>::READ_ONLY); + block_manager<>::ptr bm = open_bm(*f.metadata_dev, block_manager<>::READ_WRITE); metadata md(bm, metadata::OPEN); - aio_engine engine(f.cache_size / md.sb_.data_block_size); + + // FIXME: we're going to have to copy runs to get the through put with small block sizes + unsigned max_ios = f.cache_size / (md.sb_.data_block_size << SECTOR_SHIFT); + aio_engine engine(max_ios); copier c(engine, *f.fast_dev, *f.origin_dev, md.sb_.data_block_size, f.cache_size); - copy_visitor cv(c, clean_shutdown(md)); + + auto bar = create_progress_bar("Copying data"); + copy_visitor cv(c, f.sort_buffers, clean_shutdown(md), f.list_failed_blocks, + *bar, md.sb_.cache_blocks); + ignore_damage_visitor dv; + walk_mapping_array(*md.mappings_, cv, dv); + cv.complete(); auto stats = cv.get_stats(); - cout << stats.blocks_issued << " copies issued\n" - << stats.blocks_failed << " copies failed\n"; + cout << stats.blocks_issued - stats.blocks_failed << "/" + << stats.blocks_issued << " blocks successfully copied.\n"; if (stats.blocks_failed) - cout << stats.sectors_failed << " sectors were not copied\n"; + cout << stats.blocks_failed << " blocks were not copied\n"; - if (dv.was_corruption()) + if (dv.was_corruption()) { cout << "Metadata corruption was found, some data may not have been copied.\n"; + if (f.update_metadata) + cout << "Unable to update metadata.\n"; + + } else if (f.update_metadata) + update_metadata(md, cv.failed_writebacks()); return (stats.blocks_failed || dv.was_corruption()) ? 1 : 0; } @@ -169,6 +362,9 @@ cache_writeback_cmd::usage(std::ostream &out) const << "\t\t--metadata-device \n" << "\t\t--origin-device \n" << "\t\t--fast-device \n" + << "\t\t--buffer-size-meg \n" + << "\t\t--list-failed-blocks\n" + << "\t\t--no-metadata-update\n" << "Options:\n" << " {-h|--help}\n" << " {-V|--version}" << endl; @@ -184,6 +380,9 @@ cache_writeback_cmd::run(int argc, char **argv) { "metadata-device", required_argument, NULL, 0 }, { "origin-device", required_argument, NULL, 1 }, { "fast-device", required_argument, NULL, 2 }, + { "buffer-size-meg", required_argument, NULL, 3 }, + { "list-failed-blocks", no_argument, NULL, 4 }, + { "no-metadata-update", no_argument, NULL, 5 }, { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } @@ -203,6 +402,18 @@ cache_writeback_cmd::run(int argc, char **argv) fs.fast_dev = optarg; break; + case 3: + fs.cache_size = parse_uint64(optarg, "buffer size") * 1024 * 1024; + break; + + case 4: + fs.list_failed_blocks = true; + break; + + case 5: + fs.update_metadata = false; + break; + case 'h': usage(cout); return 0; @@ -217,6 +428,8 @@ cache_writeback_cmd::run(int argc, char **argv) } } + fs.calc_sort_buffer_size(); + if (argc != optind) { usage(cerr); return 1; @@ -241,7 +454,6 @@ cache_writeback_cmd::run(int argc, char **argv) } return writeback(fs); - } //---------------------------------------------------------------- From e8f1bda1a5a4fffd6bc6180c1959e622613177f6 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 19 Jun 2016 21:35:11 +0800 Subject: [PATCH 108/118] [thin] store the device id in single_mapping_tree_damage_visitor for error reporting 1. fix the damage type for single_mapping_tree_damage_visitor 2. walk_mapping_tree() now requires the device id 3. update metadata_dumper and thin_ls for the new walk_mapping_tree() --- thin-provisioning/mapping_tree.cc | 15 ++++++++++----- thin-provisioning/mapping_tree.h | 2 ++ thin-provisioning/metadata_dumper.cc | 9 +++++---- thin-provisioning/thin_ls.cc | 4 ++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/thin-provisioning/mapping_tree.cc b/thin-provisioning/mapping_tree.cc index 454e85c..c1a9358 100644 --- a/thin-provisioning/mapping_tree.cc +++ b/thin-provisioning/mapping_tree.cc @@ -193,14 +193,16 @@ namespace { class single_mapping_tree_damage_visitor { public: - single_mapping_tree_damage_visitor(damage_visitor &v) - : v_(v) { + single_mapping_tree_damage_visitor(damage_visitor &v, + uint64_t dev_id) + : v_(v), + dev_id_(dev_id) { } virtual void visit(btree_path const &path, btree_detail::damage const &d) { switch (path.size()) { case 0: - v_.visit(missing_devices(d.desc_, d.lost_keys_)); + v_.visit(missing_mappings(d.desc_, dev_id_, d.lost_keys_)); break; default: @@ -210,6 +212,7 @@ namespace { private: damage_visitor &v_; + uint64_t dev_id_; }; } @@ -250,19 +253,21 @@ thin_provisioning::check_mapping_tree(mapping_tree const &tree, void thin_provisioning::walk_mapping_tree(single_mapping_tree const &tree, + uint64_t dev_id, mapping_tree_detail::mapping_visitor &mv, mapping_tree_detail::damage_visitor &dv) { - single_mapping_tree_damage_visitor ll_dv(dv); + single_mapping_tree_damage_visitor ll_dv(dv, dev_id); btree_visit_values(tree, mv, ll_dv); } void thin_provisioning::check_mapping_tree(single_mapping_tree const &tree, + uint64_t dev_id, mapping_tree_detail::damage_visitor &visitor) { noop_block_time_visitor mv; - walk_mapping_tree(tree, mv, visitor); + walk_mapping_tree(tree, dev_id, mv, visitor); } //---------------------------------------------------------------- diff --git a/thin-provisioning/mapping_tree.h b/thin-provisioning/mapping_tree.h index d417b47..4e6adda 100644 --- a/thin-provisioning/mapping_tree.h +++ b/thin-provisioning/mapping_tree.h @@ -139,9 +139,11 @@ namespace thin_provisioning { mapping_tree_detail::damage_visitor &visitor); void walk_mapping_tree(single_mapping_tree const &tree, + uint64_t dev_id, mapping_tree_detail::mapping_visitor &mv, mapping_tree_detail::damage_visitor &dv); void check_mapping_tree(single_mapping_tree const &tree, + uint64_t dev_id, mapping_tree_detail::damage_visitor &visitor); } diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index ba29bfa..bbb5be8 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -196,7 +196,7 @@ namespace { try { if (!opts_.skip_mappings_) - emit_mappings(tree_root); + emit_mappings(dev_id, tree_root); } catch (exception &e) { cerr << e.what(); e_->end_device(); @@ -213,11 +213,11 @@ namespace { } private: - void emit_mappings(block_address subtree_root) { + void emit_mappings(uint64_t dev_id, block_address subtree_root) { mapping_emitter me(e_); single_mapping_tree tree(*md_->tm_, subtree_root, mapping_tree_detail::block_time_ref_counter(md_->data_sm_)); - walk_mapping_tree(tree, static_cast(me), *damage_policy_); + walk_mapping_tree(tree, dev_id, static_cast(me), *damage_policy_); } dump_options const &opts_; @@ -274,7 +274,8 @@ thin_provisioning::metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool mapping_emitter me(e); single_mapping_tree tree(*md->tm_, subtree_root, mapping_tree_detail::block_time_ref_counter(md->data_sm_)); - walk_mapping_tree(tree, static_cast(me), + // FIXME: pass the current device id instead of zero + walk_mapping_tree(tree, 0, static_cast(me), *mapping_damage_policy(repair)); } diff --git a/thin-provisioning/thin_ls.cc b/thin-provisioning/thin_ls.cc index 4e0b97d..03b8d63 100644 --- a/thin-provisioning/thin_ls.cc +++ b/thin-provisioning/thin_ls.cc @@ -248,7 +248,7 @@ namespace { mapping_pass1 pass1(mappings); fatal_mapping_damage dv; - walk_mapping_tree(dev_mappings, pass1, dv); + walk_mapping_tree(dev_mappings, dev_id, pass1, dv); } @@ -264,7 +264,7 @@ namespace { mapping_pass2 pass2(mappings); fatal_mapping_damage dv; - walk_mapping_tree(dev_mappings, pass2, dv); + walk_mapping_tree(dev_mappings, dev_id, pass2, dv); return pass2.get_exclusives(); } From 60eb60882539e5ba80f69823cca7261d5f8935eb Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 19 Jun 2016 21:35:58 +0800 Subject: [PATCH 109/118] [counting_visitor] fix the path for ValueCounter --- persistent-data/data-structures/btree_counter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index ec74e86..6ccf03a 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -37,7 +37,7 @@ namespace persistent_data { for (unsigned i = 0; i < nr; i++) { // FIXME: confirm l2 is correct node_location l2(l); - l2.push_key(i); + l2.push_key(n.key_at(i)); vc_.visit(l2, n.value_at(i)); } From 9e7af6b677853e0a5e56e05ffdc4e7c30bef8ff2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 19 Jun 2016 21:37:00 +0800 Subject: [PATCH 110/118] [metadata_counter] remove explicit try/catch when counting data space map Unlike metadata_index_store, the constructor of btree_index_store doesn't throw exceptions. --- thin-provisioning/metadata_counter.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/thin-provisioning/metadata_counter.cc b/thin-provisioning/metadata_counter.cc index e36fde6..f6055f0 100644 --- a/thin-provisioning/metadata_counter.cc +++ b/thin-provisioning/metadata_counter.cc @@ -41,12 +41,10 @@ void thin_provisioning::count_space_maps(transaction_manager::ptr tm, } // Count the data space map (no-throw) - try { + { persistent_space_map::ptr data_sm = open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); data_sm->count_metadata(bc); - } catch (std::exception &e) { - cerr << e.what() << endl; } } From 3439dbfdfc6b54eb2b064708f951f62e118e2810 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 19 Jun 2016 22:05:58 +0800 Subject: [PATCH 111/118] [metadata_counter] hide count_trees() and count_space_maps() --- thin-provisioning/metadata_counter.cc | 72 ++++++++++++++------------- thin-provisioning/metadata_counter.h | 6 --- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/thin-provisioning/metadata_counter.cc b/thin-provisioning/metadata_counter.cc index f6055f0..a928056 100644 --- a/thin-provisioning/metadata_counter.cc +++ b/thin-provisioning/metadata_counter.cc @@ -7,46 +7,50 @@ using namespace thin_provisioning; //---------------------------------------------------------------- -void thin_provisioning::count_trees(transaction_manager::ptr tm, - superblock_detail::superblock &sb, - block_counter &bc) { +namespace { + void count_trees(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc) { - // Count the device tree - { - noop_value_counter vc; - device_tree dtree(*tm, sb.device_details_root_, - device_tree_detail::device_details_traits::ref_counter()); - count_btree_blocks(dtree, bc, vc); + // Count the device tree + { + noop_value_counter vc; + device_tree dtree(*tm, sb.device_details_root_, + device_tree_detail::device_details_traits::ref_counter()); + count_btree_blocks(dtree, bc, vc); + } + + // Count the mapping tree + { + noop_value_counter vc; + mapping_tree mtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::block_traits::ref_counter(space_map::ptr())); + count_btree_blocks(mtree, bc, vc); + } } - // Count the mapping tree - { - noop_value_counter vc; - mapping_tree mtree(*tm, sb.data_mapping_root_, - mapping_tree_detail::block_traits::ref_counter(space_map::ptr())); - count_btree_blocks(mtree, bc, vc); + void count_space_maps(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc) { + // Count the metadata space map (no-throw) + try { + persistent_space_map::ptr metadata_sm = + open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); + metadata_sm->count_metadata(bc); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + // Count the data space map (no-throw) + { + persistent_space_map::ptr data_sm = + open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); + data_sm->count_metadata(bc); + } } } -void thin_provisioning::count_space_maps(transaction_manager::ptr tm, - superblock_detail::superblock &sb, - block_counter &bc) { - // Count the metadata space map (no-throw) - try { - persistent_space_map::ptr metadata_sm = - open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); - metadata_sm->count_metadata(bc); - } catch (std::exception &e) { - cerr << e.what() << endl; - } - - // Count the data space map (no-throw) - { - persistent_space_map::ptr data_sm = - open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); - data_sm->count_metadata(bc); - } -} +//---------------------------------------------------------------- void thin_provisioning::count_metadata(transaction_manager::ptr tm, superblock_detail::superblock &sb, diff --git a/thin-provisioning/metadata_counter.h b/thin-provisioning/metadata_counter.h index 861d72f..67801e8 100644 --- a/thin-provisioning/metadata_counter.h +++ b/thin-provisioning/metadata_counter.h @@ -7,12 +7,6 @@ //---------------------------------------------------------------- namespace thin_provisioning { - void count_trees(transaction_manager::ptr tm, - superblock_detail::superblock &sb, - block_counter &bc); - void count_space_maps(transaction_manager::ptr tm, - superblock_detail::superblock &sb, - block_counter &bc); void count_metadata(transaction_manager::ptr tm, superblock_detail::superblock &sb, block_counter &bc, From 7eac48793caf1146bd2ff9937a99e9fb7c031e18 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Mon, 20 Jun 2016 00:40:10 +0800 Subject: [PATCH 112/118] [space map disk] tidy up: add const qualifier --- persistent-data/space-maps/disk.cc | 4 ++-- persistent-data/space-maps/disk.h | 4 ++-- thin-provisioning/metadata_counter.cc | 10 +++++----- thin-provisioning/metadata_counter.h | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/persistent-data/space-maps/disk.cc b/persistent-data/space-maps/disk.cc index 9c51921..caeaca3 100644 --- a/persistent-data/space-maps/disk.cc +++ b/persistent-data/space-maps/disk.cc @@ -742,7 +742,7 @@ persistent_data::create_disk_sm(transaction_manager &tm, } checked_space_map::ptr -persistent_data::open_disk_sm(transaction_manager &tm, void *root) +persistent_data::open_disk_sm(transaction_manager &tm, void const *root) { sm_root_disk d; sm_root v; @@ -770,7 +770,7 @@ persistent_data::create_metadata_sm(transaction_manager &tm, block_address nr_bl } checked_space_map::ptr -persistent_data::open_metadata_sm(transaction_manager &tm, void *root) +persistent_data::open_metadata_sm(transaction_manager &tm, void const *root) { sm_root_disk d; sm_root v; diff --git a/persistent-data/space-maps/disk.h b/persistent-data/space-maps/disk.h index 9edef6b..906221e 100644 --- a/persistent-data/space-maps/disk.h +++ b/persistent-data/space-maps/disk.h @@ -29,13 +29,13 @@ namespace persistent_data { create_disk_sm(transaction_manager &tm, block_address nr_blocks); checked_space_map::ptr - open_disk_sm(transaction_manager &tm, void *root); + open_disk_sm(transaction_manager &tm, void const *root); checked_space_map::ptr create_metadata_sm(transaction_manager &tm, block_address nr_blocks); checked_space_map::ptr - open_metadata_sm(transaction_manager &tm, void *root); + open_metadata_sm(transaction_manager &tm, void const *root); bcache::validator::ptr bitmap_validator(); diff --git a/thin-provisioning/metadata_counter.cc b/thin-provisioning/metadata_counter.cc index a928056..95487d2 100644 --- a/thin-provisioning/metadata_counter.cc +++ b/thin-provisioning/metadata_counter.cc @@ -9,7 +9,7 @@ using namespace thin_provisioning; namespace { void count_trees(transaction_manager::ptr tm, - superblock_detail::superblock &sb, + superblock_detail::superblock const &sb, block_counter &bc) { // Count the device tree @@ -30,12 +30,12 @@ namespace { } void count_space_maps(transaction_manager::ptr tm, - superblock_detail::superblock &sb, + superblock_detail::superblock const &sb, block_counter &bc) { // Count the metadata space map (no-throw) try { persistent_space_map::ptr metadata_sm = - open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); + open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); metadata_sm->count_metadata(bc); } catch (std::exception &e) { cerr << e.what() << endl; @@ -44,7 +44,7 @@ namespace { // Count the data space map (no-throw) { persistent_space_map::ptr data_sm = - open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); + open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); data_sm->count_metadata(bc); } } @@ -53,7 +53,7 @@ namespace { //---------------------------------------------------------------- void thin_provisioning::count_metadata(transaction_manager::ptr tm, - superblock_detail::superblock &sb, + superblock_detail::superblock const &sb, block_counter &bc, bool skip_metadata_snap) { // Count the superblock diff --git a/thin-provisioning/metadata_counter.h b/thin-provisioning/metadata_counter.h index 67801e8..bc65ab9 100644 --- a/thin-provisioning/metadata_counter.h +++ b/thin-provisioning/metadata_counter.h @@ -8,7 +8,7 @@ namespace thin_provisioning { void count_metadata(transaction_manager::ptr tm, - superblock_detail::superblock &sb, + superblock_detail::superblock const &sb, block_counter &bc, bool skip_metadata_snap = false); } From e6562c611013d412c2047e114df0b1029f16b66d Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 15 Jul 2016 11:23:07 +0100 Subject: [PATCH 113/118] thin_generate_metadata --- bin/thin_generate_metadata | 1 + thin-provisioning/thin_generate_metadata.cc | 196 +++++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) create mode 120000 bin/thin_generate_metadata diff --git a/bin/thin_generate_metadata b/bin/thin_generate_metadata new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_generate_metadata @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file diff --git a/thin-provisioning/thin_generate_metadata.cc b/thin-provisioning/thin_generate_metadata.cc index 4d5903d..6eb3c62 100644 --- a/thin-provisioning/thin_generate_metadata.cc +++ b/thin-provisioning/thin_generate_metadata.cc @@ -1,9 +1,129 @@ -#include "thin-provisioning/commands.h" +// Copyright (C) 2016 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . +#include "base/output_file_requirements.h" +#include "persistent-data/file_utils.h" +#include "thin-provisioning/commands.h" +#include "thin-provisioning/emitter.h" +#include "thin-provisioning/human_readable_format.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/restore_emitter.h" +#include "thin-provisioning/xml_format.h" +#include "version.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace boost; +using namespace persistent_data; +using namespace std; using namespace thin_provisioning; //---------------------------------------------------------------- +namespace { + struct flags { + flags() + : data_block_size(128), + nr_data_blocks(10240), + nr_thins(1), + blocks_per_thin(1024), + run_lengths(1024) { + } + + block_address data_block_size; + block_address nr_data_blocks; + unsigned nr_thins; + block_address blocks_per_thin; + block_address run_lengths; + optional output; + }; + + // This is how we stir in some entropy. It mixes up the data + // device. + class shuffler { + public: + shuffler(block_address nr_blocks, unsigned run_lengths) + : nr_blocks_(nr_blocks / run_lengths), + run_lengths_(run_lengths) { + } + + block_address map(block_address b) const { + return reverse(b / run_lengths_) + (b % run_lengths_); + } + + private: + block_address reverse(block_address b) const { + return nr_blocks_ - b - 1ull; + } + + block_address nr_blocks_; + block_address run_lengths_; + }; + + void generate_device(emitter::ptr e, shuffler const &s, uint32_t dev_id, + block_address nr_blocks, block_address base) { + + e->begin_device(dev_id, nr_blocks, 0, 0, 0); + for (unsigned b = 0; b < nr_blocks; b++) + e->single_map(b, s.map(base + b), 0); + e->end_device(); + } + + void generate_metadata(flags const &fs, emitter::ptr e) { + e->begin_superblock("fake metadata", 0, 0, optional(), optional(), + fs.data_block_size, fs.nr_data_blocks, optional()); + + shuffler s(fs.nr_data_blocks, fs.run_lengths); + for (unsigned i = 0; i < fs.nr_thins; i++) + generate_device(e, s, i, fs.blocks_per_thin, i * fs.blocks_per_thin); + + e->end_superblock(); + } + + int create_metadata(flags const &fs) { + try { + // The block size gets updated by the restorer. + block_manager<>::ptr bm(open_bm(*fs.output, block_manager<>::READ_WRITE)); + metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0)); + emitter::ptr restorer = create_restore_emitter(md); + + generate_metadata(fs, restorer); + + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + + return 0; + } +} + +//---------------------------------------------------------------- + thin_generate_metadata_cmd::thin_generate_metadata_cmd() : command("thin_generate_metadata") { @@ -12,14 +132,84 @@ thin_generate_metadata_cmd::thin_generate_metadata_cmd() void thin_generate_metadata_cmd::usage(std::ostream &out) const { - + out << "Usage: " << get_name() << " [options]\n" + << "Options:\n" + << " {-h|--help}\n" + << " --data-block-size \n" + << " --nr-data-blocks \n" + << " --nr-thins \n" + << " --blocks-per-thin \n" + << " --run-lengths \n" + << " {-o|--output} \n" + << " {-V|--version}" << endl; } int thin_generate_metadata_cmd::run(int argc, char **argv) { + int c; + struct flags fs; + const char *shortopts = "hi:o:qV"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "output", required_argument, NULL, 'o'}, + { "data-block-size", required_argument, NULL, 1}, + { "nr-data-blocks", required_argument, NULL, 2}, + { "nr-thins", required_argument, NULL, 3}, + { "blocks-per-thin", required_argument, NULL, 4}, + { "run-lengths", required_argument, NULL, 5}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; - return 1; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'o': + fs.output = optarg; + break; + + case 1: + fs.data_block_size = parse_uint64(optarg, "data block size"); + break; + + case 2: + fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks"); + break; + + case 3: + fs.nr_thins = parse_uint64(optarg, "nr thins"); + break; + + case 4: + fs.blocks_per_thin = parse_uint64(optarg, "blocks per thin"); + break; + + case 5: + fs.run_lengths = parse_uint64(optarg, "run lengths"); + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr); + return 1; + } + } + + if (!fs.output) { + cerr << "No output file provided.\n\n"; + usage(cerr); + return 1; + } else + check_output_file_requirements(*fs.output); + + return create_metadata(fs); } //---------------------------------------------------------------- From 702a38f43882dce419751913907d6c714f9336ba Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 21 Jul 2016 15:42:10 +0100 Subject: [PATCH 114/118] [many tools] fix bug in previous patch --- base/output_file_requirements.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/base/output_file_requirements.cc b/base/output_file_requirements.cc index 001e127..fefdc0f 100644 --- a/base/output_file_requirements.cc +++ b/base/output_file_requirements.cc @@ -37,14 +37,17 @@ base::check_output_file_requirements(string const &path) explain_output_file_requirements(); } - if (!info.st_size) { - cerr << "Zero size output file.\n\n"; - explain_output_file_requirements(); - } + // We only really want these checks for regular files + if (S_ISREG(info.st_mode)) { + if (!info.st_size) { + cerr << "Zero size output file.\n\n"; + explain_output_file_requirements(); + } - if (info.st_size < MIN_SIZE) { - cerr << "Output file too small.\n\n"; - explain_output_file_requirements(); + if (info.st_size < MIN_SIZE) { + cerr << "Output file too small.\n\n"; + explain_output_file_requirements(); + } } } From 6863db6f8682995f2d50458d7ec9dd22d52166e4 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 10 Aug 2016 23:40:48 +0800 Subject: [PATCH 115/118] [thin] fix duplicated counting of mapped blocks in restore emitter --- persistent-data/data-structures/btree.h | 2 +- persistent-data/data-structures/btree.tcc | 4 +++- thin-provisioning/restore_emitter.cc | 5 ++--- thin-provisioning/thin_pool.cc | 2 +- thin-provisioning/thin_pool.h | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/persistent-data/data-structures/btree.h b/persistent-data/data-structures/btree.h index b91fb20..9e85a36 100644 --- a/persistent-data/data-structures/btree.h +++ b/persistent-data/data-structures/btree.h @@ -322,7 +322,7 @@ namespace persistent_data { maybe_pair lookup_le(key const &key) const; maybe_pair lookup_ge(key const &key) const; - void insert(key const &key, typename ValueTraits::value_type const &value); + bool insert(key const &key, typename ValueTraits::value_type const &value); void remove(key const &key); void set_root(block_address root); diff --git a/persistent-data/data-structures/btree.tcc b/persistent-data/data-structures/btree.tcc index 3fc96e4..ce49a30 100644 --- a/persistent-data/data-structures/btree.tcc +++ b/persistent-data/data-structures/btree.tcc @@ -498,7 +498,7 @@ namespace persistent_data { } template - void + bool btree:: insert(key const &key, typename ValueTraits::value_type const &value) @@ -531,6 +531,8 @@ namespace persistent_data { n.set_value(index, value); root_ = spine.get_root(); + + return need_insert; } template diff --git a/thin-provisioning/restore_emitter.cc b/thin-provisioning/restore_emitter.cc index c16eb70..46b7f92 100644 --- a/thin-provisioning/restore_emitter.cc +++ b/thin-provisioning/restore_emitter.cc @@ -136,10 +136,9 @@ namespace { mapping_tree_detail::block_time bt; bt.block_ = data_block; bt.time_ = time; - current_mapping_->insert(key, bt); + current_device_details_.mapped_blocks_ += + static_cast(current_mapping_->insert(key, bt)); md_->data_sm_->inc(data_block); - - current_device_details_.mapped_blocks_ += 1; } private: diff --git a/thin-provisioning/thin_pool.cc b/thin-provisioning/thin_pool.cc index 1abaa57..d133711 100644 --- a/thin-provisioning/thin_pool.cc +++ b/thin-provisioning/thin_pool.cc @@ -50,7 +50,7 @@ thin::lookup(block_address thin_block) return pool_->md_->mappings_->lookup(key); } -void +bool thin::insert(block_address thin_block, block_address data_block) { uint64_t key[2] = {dev_, thin_block}; diff --git a/thin-provisioning/thin_pool.h b/thin-provisioning/thin_pool.h index 66ae307..0c6b156 100644 --- a/thin-provisioning/thin_pool.h +++ b/thin-provisioning/thin_pool.h @@ -39,7 +39,7 @@ namespace thin_provisioning { thin_dev_t get_dev_t() const; maybe_address lookup(block_address thin_block); - void insert(block_address thin_block, block_address data_block); + bool insert(block_address thin_block, block_address data_block); void remove(block_address thin_block); void set_snapshot_time(uint32_t time); From 45dbc2593aa5f9cdd7fa05b7c6925d49530cee08 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 10 Aug 2016 23:41:25 +0800 Subject: [PATCH 116/118] [thin_ll_restore] check preallocation of output file --- thin-provisioning/thin_ll_restore.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/thin_ll_restore.cc b/thin-provisioning/thin_ll_restore.cc index 4082e0b..ff62298 100644 --- a/thin-provisioning/thin_ll_restore.cc +++ b/thin-provisioning/thin_ll_restore.cc @@ -14,6 +14,7 @@ // with thin-provisioning-tools. If not, see // . +#include "base/output_file_requirements.h" #include "base/xml_utils.h" #include "metadata_dumper.h" #include "metadata.h" @@ -261,12 +262,25 @@ thin_ll_restore_cmd::run(int argc, char **argv) { return 1; } - if (!input_metadata.length() || !input.length() || !output.length()) { - cerr << "No input/output file provided." << endl; + if (!input.length()) { + cerr << "No input file provided." << endl; usage(cerr); return 1; } + if (!input_metadata.length()) { + cerr << "No input metadata provided." << endl; + usage(cerr); + return 1; + } + + if (!output.length()) { + cerr << "No output file provided." << endl; + usage(cerr); + return 1; + } else + check_output_file_requirements(output); + return low_level_restore(input_metadata, input, output, f); } From 0c78d8d0c563856d2a5c1062e5517f7e731c0e1e Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Wed, 7 Sep 2016 13:16:32 +0000 Subject: [PATCH 117/118] fix compilation for gcc 5.4.0 --- caching/cache_writeback.cc | 1 + contrib/Makefile.in | 2 +- thin-provisioning/static_library_emitter.cc | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc index 7dbf4c3..be7a9a7 100644 --- a/caching/cache_writeback.cc +++ b/caching/cache_writeback.cc @@ -10,6 +10,7 @@ #include #include #include +#include using namespace bcache; using namespace caching; diff --git a/contrib/Makefile.in b/contrib/Makefile.in index 0b6c28f..c093ca7 100644 --- a/contrib/Makefile.in +++ b/contrib/Makefile.in @@ -12,7 +12,7 @@ contrib: $(PLUGINS) $(PLUGIN_LIBS) contrib/%.o: contrib/%.cc $(V)echo " [CC] $@" - $(V)$(CC) $^ -o $@ + $(V)$(CXX) $(INCLUDES) $(CXXFLAGS) $^ -c -o $@ contrib/%.a: contrib/%.o $(V)echo " [AR] $@" diff --git a/thin-provisioning/static_library_emitter.cc b/thin-provisioning/static_library_emitter.cc index 8caa507..184a458 100644 --- a/thin-provisioning/static_library_emitter.cc +++ b/thin-provisioning/static_library_emitter.cc @@ -1,6 +1,7 @@ #include "thin-provisioning/shared_library_emitter.h" #include #include "contrib/tmakatos_emitter.h" +#include using namespace std; using namespace thin_provisioning; From 73e583e662af5bfb8817486209093e4ea764c871 Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Mon, 6 Feb 2017 13:11:10 +0300 Subject: [PATCH 118/118] remove debug print --- thin-provisioning/static_library_emitter.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/thin-provisioning/static_library_emitter.cc b/thin-provisioning/static_library_emitter.cc index 184a458..973ec17 100644 --- a/thin-provisioning/static_library_emitter.cc +++ b/thin-provisioning/static_library_emitter.cc @@ -14,9 +14,6 @@ thin_provisioning::create_custom_emitter(string const &shared_lib, ostream &out) if (shared_lib != "tmakatos_emitter.so") throw runtime_error(shared_lib + ": no such emitter"); - cout << "XXX creating tmakatos_emitter" << endl; - cout << flush; - return emitter::ptr(new tmakatos_emitter::binary_emitter(out)); }