emo
/
thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/github-m-h-tsai/v0.6.2-repairtool' into 2016-02-29-mingus-merge 

Conflicts:
	Makefile.in
	VERSION
	thin-provisioning/commands.cc
	thin-provisioning/commands.h
	thin-provisioning/thin_delta.cc
This commit is contained in:
Joe Thornber 2016-02-29 10:29:28 +00:00
parent e78de5d3ad 08a7093cd5
commit 286f70ceac
32 changed files with 1592 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGES
View File

@ -1,8 +1,9 @@
v0.6.2
======
Fix recent regression in thin_repair.
Force g++-98 dialect
- Fix bug in thin_delta
- Fix recent regression in thin_repair.
- Force g++-98 dialect
v0.6.1
======

6
Makefile.in
View File

@ -85,6 +85,7 @@ SOURCE=\
thin-provisioning/mapping_tree.cc \
thin-provisioning/metadata.cc \
thin-provisioning/metadata_checker.cc \
thin-provisioning/metadata_counter.cc \
thin-provisioning/metadata_dumper.cc \
thin-provisioning/pool_stream.cc \
thin-provisioning/restore_emitter.cc \
@ -99,14 +100,17 @@ SOURCE=\
thin-provisioning/thin_repair.cc \
thin-provisioning/thin_restore.cc \
thin-provisioning/thin_rmap.cc \
thin-provisioning/thin_show_duplicates.cc \
thin-provisioning/thin_trim.cc \
thin-provisioning/xml_format.cc
DEVTOOLS_SOURCE=\
thin-provisioning/thin_ll_dump.cc \
thin-provisioning/thin_ll_restore.cc \
thin-provisioning/thin_show_duplicates.cc \
thin-provisioning/thin_generate_metadata.cc \
thin-provisioning/variable_chunk_stream.cc \
thin-provisioning/thin_show_metadata.cc \
thin-provisioning/thin_scan.cc \
ui/ui.cc
ifeq ("@DEVTOOLS@", "yes")

7
base/xml_utils.cc
View File

@ -19,13 +19,16 @@ xml_parser::parse(std::string const &backup_file, bool quiet)
size_t total = 0;
size_t input_length = get_file_length(backup_file);
while (!in.eof()) {
XML_Error error_code = XML_ERROR_NONE;
while (!in.eof() && error_code == XML_ERROR_NONE) {
char buffer[4096];
in.read(buffer, sizeof(buffer));
size_t len = in.gcount();
int done = in.eof();
if (!XML_Parse(parser_, buffer, len, done)) {
// Do not throw while normally aborted by element handlers
if (!XML_Parse(parser_, buffer, len, done) &&
(error_code = XML_GetErrorCode(parser_)) != XML_ERROR_ABORTED) {
ostringstream out;
out << "Parse error at line "
<< XML_GetCurrentLineNumber(parser_)

4
block-cache/block_cache.cc
View File

@ -545,7 +545,9 @@ block_cache::get(block_address index, unsigned flags, validator::ptr v)
return *b;
}
throw std::runtime_error("couldn't get block");
std::ostringstream out;
out << "couldn't get block " << index;
throw std::runtime_error(out.str());
}
void

2
block-cache/block_cache.h
View File

@ -32,12 +32,14 @@ namespace bcache {
virtual ~validator() {}
virtual void check(void const *data, block_address location) const = 0;
virtual bool check_raw(void const *data) const = 0;
virtual void prepare(void *data, block_address location) const = 0;
};
class noop_validator : public validator {
public:
void check(void const *data, block_address location) const {}
bool check_raw(void const *data) const {return true;}
void prepare(void *data, block_address location) const {}
};

9
caching/superblock.cc
View File

@ -292,6 +292,15 @@ namespace validator {
throw checksum_error("bad checksum in superblock");
}
virtual bool check_raw(void const *raw) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(sbd->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
superblock_disk *sbd = reinterpret_cast<superblock_disk *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);

9
era/superblock.cc
View File

@ -219,6 +219,15 @@ namespace era_validator {
throw checksum_error("bad checksum in superblock");
}
virtual bool check_raw(void const *raw) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(sbd->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
superblock_disk *sbd = reinterpret_cast<superblock_disk *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);

30
persistent-data/block_counter.h
View File

@ -20,6 +20,7 @@
#define BLOCK_COUNTER_H
#include "block.h"
#include "run_set.h"
//----------------------------------------------------------------
@ -32,7 +33,9 @@ namespace persistent_data {
public:
typedef std::map<block_address, unsigned> count_map;
void inc(block_address b) {
virtual ~block_counter() {}
virtual void inc(block_address b) {
count_map::iterator it = counts_.find(b);
if (it == counts_.end())
counts_.insert(make_pair(b, 1));
@ -40,7 +43,7 @@ namespace persistent_data {
it->second++;
}
unsigned get_count(block_address b) const {
virtual unsigned get_count(block_address b) const {
count_map::const_iterator it = counts_.find(b);
return (it == counts_.end()) ? 0 : it->second;
}
@ -52,6 +55,29 @@ namespace persistent_data {
private:
count_map counts_;
};
//----------------------------------------------------------------
// Little helper class that keeps track of which blocks
// are referenced.
//----------------------------------------------------------------
class binary_block_counter : public block_counter {
public:
virtual ~binary_block_counter() {}
virtual void inc(block_address b) {
visited_.add(b);
}
virtual unsigned get_count(block_address b) const {
return visited_.member(b) ? 1 : 0;
}
base::run_set<block_address> const& get_visited() const {
return visited_;
}
private:
base::run_set<block_address> visited_;
};
}
//----------------------------------------------------------------

9
persistent-data/data-structures/array.h
View File

@ -43,6 +43,15 @@ namespace persistent_data {
throw checksum_error("bad block nr in array block");
}
virtual bool check_raw(void const *raw) const {
array_block_disk const *data = reinterpret_cast<array_block_disk const *>(raw);
crc32c sum(ARRAY_CSUM_XOR);
sum.append(&data->max_entries, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(data->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
array_block_disk *data = reinterpret_cast<array_block_disk *>(raw);
data->blocknr = to_disk<base::le64, uint64_t>(location);

23
persistent-data/data-structures/btree.tcc
View File

@ -32,7 +32,6 @@ namespace {
using namespace persistent_data;
using namespace btree_detail;
using namespace std;
}
//----------------------------------------------------------------
@ -90,14 +89,22 @@ namespace persistent_data {
{
uint32_t flags = to_cpu<uint32_t>(raw_->header.flags);
if (flags & INTERNAL_NODE) {
if (flags & LEAF_NODE)
throw runtime_error("btree node is both internal and leaf");
if (flags & LEAF_NODE) {
ostringstream out;
out << "btree node is both internal and leaf"
<< " (block " << location_ << ")";
throw runtime_error(out.str());
}
return INTERNAL;
} else if (flags & LEAF_NODE)
return LEAF;
else
throw runtime_error("unknown node type");
else {
ostringstream out;
out << "unknown node type"
<< " (block " << location_ << ")";
throw runtime_error(out.str());
}
}
template <typename ValueTraits>
@ -352,7 +359,8 @@ namespace persistent_data {
std::ostringstream out;
out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type)
<< ", but got " << get_value_size()
<< ". This is not the btree you are looking for." << std::endl;
<< ". This is not the btree you are looking for."
<< " (block " << location_ << ")" << std::endl;
return out.str();
}
@ -371,7 +379,8 @@ namespace persistent_data {
if (max < get_nr_entries()) {
std::ostringstream out;
out << "Bad nr of elements: max per block = "
<< max << ", actual = " << get_nr_entries() << std::endl;
<< max << ", actual = " << get_nr_entries()
<< " (block " << location_ << ")" << std::endl;
throw std::runtime_error(out.str());
}

17
persistent-data/data-structures/btree_base_visitor.h Normal file
View File

@ -0,0 +1,17 @@
#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H
#include "persistent-data/data-structures/btree.h"
namespace persistent_data {
namespace btree_detail {
template <typename ValueType>
class noop_value_visitor {
public:
virtual void visit(btree_path const &path, ValueType const &v) {
}
};
}
}
#endif

39
persistent-data/data-structures/btree_counter.h
View File

@ -2,35 +2,44 @@
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/btree_base_visitor.h"
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include "persistent-data/block_counter.h"
//----------------------------------------------------------------
namespace persistent_data {
namespace btree_count_detail {
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
class counting_visitor : public btree<Levels, ValueTraits>::visitor {
template <typename ValueVisitor, typename DamageVisitor, unsigned Levels, typename ValueTraits, typename ValueCounter>
class counting_visitor : public btree_damage_visitor<ValueVisitor, DamageVisitor, Levels, ValueTraits> {
typedef btree_damage_visitor<ValueVisitor, DamageVisitor, Levels, ValueTraits> BtreeDamageVisitor;
public:
typedef btree<Levels, ValueTraits> tree;
counting_visitor(block_counter &bc, ValueCounter &vc)
: bc_(bc),
counting_visitor(ValueVisitor &value_visitor,
DamageVisitor &damage_visitor,
block_counter &bc,
ValueCounter &vc)
: BtreeDamageVisitor(value_visitor, damage_visitor, false),
bc_(bc),
vc_(vc) {
}
virtual bool visit_internal(node_location const &l,
typename tree::internal_node const &n) {
return visit_node(n);
return BtreeDamageVisitor::visit_internal(l, n) ?
visit_node(n) : false;
}
virtual bool visit_internal_leaf(node_location const &l,
typename tree::internal_node const &n) {
return visit_node(n);
return BtreeDamageVisitor::visit_internal_leaf(l, n) ?
visit_node(n) : false;
}
virtual bool visit_leaf(node_location const &l,
typename tree::leaf_node const &n) {
if (visit_node(n)) {
if (BtreeDamageVisitor::visit_leaf(l, n) && visit_node(n)) {
unsigned nr = n.get_nr_entries();
for (unsigned i = 0; i < nr; i++) {
@ -85,7 +94,21 @@ namespace persistent_data {
// is not corrupt.
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc) {
btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc);
typedef noop_value_visitor<typename ValueTraits::value_type> NoopValueVisitor;
NoopValueVisitor noop_vv;
noop_damage_visitor noop_dv;
btree_count_detail::counting_visitor<NoopValueVisitor, noop_damage_visitor, Levels, ValueTraits, ValueCounter> v(noop_vv, noop_dv, bc, vc);
tree.visit_depth_first(v);
}
template <unsigned Levels, typename ValueTraits>
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc) {
typedef noop_value_visitor<typename ValueTraits::value_type> NoopValueVisitor;
NoopValueVisitor noop_vv;
noop_damage_visitor noop_dv;
typedef noop_value_counter<typename ValueTraits::value_type> NoopValueCounter;
NoopValueCounter vc;
btree_count_detail::counting_visitor<NoopValueVisitor, noop_damage_visitor, Levels, ValueTraits, NoopValueCounter> v(noop_vv, noop_dv, bc, vc);
tree.visit_depth_first(v);
}
}

32
persistent-data/data-structures/btree_damage_visitor.h
View File

@ -27,6 +27,12 @@ namespace persistent_data {
return out;
}
class noop_damage_visitor {
public:
virtual void visit(btree_path const &path, damage const &d) {
}
};
// Tracks damage in a single level btree. Use multiple
// trackers if you have a multilayer tree.
class damage_tracker {
@ -152,8 +158,9 @@ namespace persistent_data {
typedef boost::optional<run64> maybe_run64;
btree_damage_visitor(ValueVisitor &value_visitor,
DamageVisitor &damage_visitor)
: avoid_repeated_visits_(true),
DamageVisitor &damage_visitor,
bool avoid_repeated_visits = true)
: avoid_repeated_visits_(avoid_repeated_visits),
value_visitor_(value_visitor),
damage_visitor_(damage_visitor) {
}
@ -300,14 +307,16 @@ namespace persistent_data {
size_t elt_size = sizeof(uint64_t) + n.get_value_size();
if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) {
std::ostringstream out;
out << "max entries too large: " << n.get_max_entries();
out << "max entries too large: " << n.get_max_entries()
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
if (n.get_max_entries() % 3) {
std::ostringstream out;
out << "max entries is not divisible by 3: " << n.get_max_entries();
out << "max entries is not divisible by 3: " << n.get_max_entries()
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@ -321,7 +330,8 @@ namespace persistent_data {
std::ostringstream out;
out << "bad nr_entries: "
<< n.get_nr_entries() << " < "
<< n.get_max_entries();
<< n.get_max_entries()
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@ -333,7 +343,8 @@ namespace persistent_data {
<< n.get_nr_entries()
<< ", expected at least "
<< min
<< "(max_entries = " << n.get_max_entries() << ")";
<< " (block " << n.get_location()
<< ", max_entries = " << n.get_max_entries() << ")";
report_damage(out.str());
return false;
}
@ -354,7 +365,8 @@ namespace persistent_data {
uint64_t k = n.key_at(i);
if (k <= last_key) {
ostringstream out;
out << "keys are out of order, " << k << " <= " << last_key;
out << "keys are out of order, " << k << " <= " << last_key
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@ -372,7 +384,8 @@ namespace persistent_data {
if (*key > n.key_at(0)) {
ostringstream out;
out << "parent key mismatch: parent was " << *key
<< ", but lowest in node was " << n.key_at(0);
<< ", but lowest in node was " << n.key_at(0)
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@ -388,7 +401,8 @@ namespace persistent_data {
if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) {
ostringstream out;
out << "the last key of the previous leaf was " << *last_leaf_key_[level]
<< " and the first key of this leaf is " << n.key_at(0);
<< " and the first key of this leaf is " << n.key_at(0)
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}

39
persistent-data/data-structures/btree_key_value_extractor.h Normal file
View File

@ -0,0 +1,39 @@
#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include <map>
namespace persistent_data {
namespace btree_detail {
template <typename ValueType>
class key_value_extractor {
typedef typename std::map<uint64_t, ValueType> MapType;
public:
key_value_extractor(MapType &map): map_(map) {
}
virtual ~key_value_extractor() {
}
virtual void visit(btree_path const &path, ValueType const &v) {
map_.insert(std::make_pair(path.back(), v));
}
private:
MapType &map_;
};
template <unsigned Levels, typename ValueTraits>
void btree_extract_key_values(btree<Levels, ValueTraits> const &tree,
std::map<uint64_t, typename ValueTraits::value_type> &map) {
typedef key_value_extractor<typename ValueTraits::value_type> KeyValueExtractor;
KeyValueExtractor kve(map);
noop_damage_visitor noop_dv;
btree_detail::btree_damage_visitor<KeyValueExtractor, noop_damage_visitor, Levels, ValueTraits>
v(kve, noop_dv);
tree.visit_depth_first(v);
}
}
}
#endif

33
persistent-data/space-maps/disk.cc
View File

@ -50,6 +50,15 @@ namespace {
throw checksum_error("bad block nr in space map bitmap");
}
virtual bool check_raw(void const *raw) const {
bitmap_header const *data = reinterpret_cast<bitmap_header const *>(raw);
crc32c sum(BITMAP_CSUM_XOR);
sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(data->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
bitmap_header *data = reinterpret_cast<bitmap_header *>(raw);
data->blocknr = to_disk<base::le64, uint64_t>(location);
@ -75,6 +84,15 @@ namespace {
throw checksum_error("bad block nr in metadata index block");
}
virtual bool check_raw(void const *raw) const {
metadata_index const *mi = reinterpret_cast<metadata_index const *>(raw);
crc32c sum(INDEX_CSUM_XOR);
sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(mi->csum_))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
metadata_index *mi = reinterpret_cast<metadata_index *>(raw);
mi->blocknr_ = to_disk<base::le64, uint64_t>(location);
@ -85,11 +103,6 @@ namespace {
}
};
bcache::validator::ptr
index_validator() {
return bcache::validator::ptr(new index_block_validator());
}
//--------------------------------
class bitmap {
@ -771,4 +784,14 @@ persistent_data::open_metadata_sm(transaction_manager &tm, void *root)
checked_space_map::ptr(new sm_disk(store, tm, v))));
}
bcache::validator::ptr
persistent_data::bitmap_validator() {
return bcache::validator::ptr(new bitmap_block_validator());
}
bcache::validator::ptr
persistent_data::index_validator() {
return bcache::validator::ptr(new index_block_validator());
}
//----------------------------------------------------------------

6
persistent-data/space-maps/disk.h
View File

@ -36,6 +36,12 @@ namespace persistent_data {
checked_space_map::ptr
open_metadata_sm(transaction_manager &tm, void *root);
bcache::validator::ptr
bitmap_validator();
bcache::validator::ptr
index_validator();
}
//----------------------------------------------------------------

10
persistent-data/validators.cc
View File

@ -31,6 +31,16 @@ namespace {
}
}
virtual bool check_raw(void const *raw) const {
disk_node const *data = reinterpret_cast<disk_node const *>(raw);
node_header const *n = &data->header;
crc32c sum(BTREE_CSUM_XOR);
sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(n->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
disk_node *data = reinterpret_cast<disk_node *>(raw);
node_header *n = &data->header;

3
thin-provisioning/commands.cc
View File

@ -11,6 +11,8 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_check_cmd()));
app.add_cmd(command::ptr(new thin_delta_cmd()));
app.add_cmd(command::ptr(new thin_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
app.add_cmd(command::ptr(new thin_ls_cmd()));
app.add_cmd(command::ptr(new thin_metadata_size_cmd()));
app.add_cmd(command::ptr(new thin_restore_cmd()));
@ -18,6 +20,7 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_rmap_cmd()));
#ifdef DEV_TOOLS
app.add_cmd(command::ptr(new thin_scan_cmd()));
app.add_cmd(command::ptr(new thin_trim_cmd()));
app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));

21
thin-provisioning/commands.h
View File

@ -28,6 +28,20 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_ll_dump_cmd : public base::command {
public:
thin_ll_dump_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ll_restore_cmd : public base::command {
public:
thin_ll_restore_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ls_cmd : public base::command {
public:
thin_ls_cmd();
@ -64,6 +78,13 @@ namespace thin_provisioning {
};
#ifdef DEV_TOOLS
class thin_scan_cmd : public base::command {
public:
thin_scan_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_trim_cmd : public base::command {
public:
thin_trim_cmd();

2
thin-provisioning/emitter.h
View File

@ -49,6 +49,8 @@ namespace thin_provisioning {
virtual void begin_superblock(std::string const &uuid,
uint64_t time,
uint64_t trans_id,
boost::optional<uint32_t> flags,
boost::optional<uint32_t> version,
uint32_t data_block_size,
uint64_t nr_data_blocks,
boost::optional<uint64_t> metadata_snap) = 0;

4
thin-provisioning/human_readable_format.cc
View File

@ -43,12 +43,16 @@ namespace {
void begin_superblock(string const &uuid,
uint64_t time,
uint64_t trans_id,
boost::optional<uint32_t> flags,
boost::optional<uint32_t> version,
uint32_t data_block_size,
uint64_t nr_data_blocks,
boost::optional<uint64_t> metadata_snap) {
out_ << "begin superblock: \"" << uuid << "\""
<< ", " << time
<< ", " << trans_id
<< ", " << (flags ? *flags : 0)
<< ", " << (version ? *version : 1)
<< ", " << data_block_size
<< ", " << nr_data_blocks;
if (metadata_snap)

73
thin-provisioning/metadata_counter.cc Normal file
View File

@ -0,0 +1,73 @@
#include "thin-provisioning/metadata_counter.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/space-maps/disk_structures.h"
using namespace persistent_data;
using namespace thin_provisioning;
//----------------------------------------------------------------
void thin_provisioning::count_trees(transaction_manager::ptr tm,
superblock_detail::superblock &sb,
block_counter &bc) {
// Count the device tree
{
noop_value_counter<device_tree_detail::device_details> vc;
device_tree dtree(*tm, sb.device_details_root_,
device_tree_detail::device_details_traits::ref_counter());
count_btree_blocks(dtree, bc, vc);
}
// Count the mapping tree
{
noop_value_counter<mapping_tree_detail::block_time> vc;
mapping_tree mtree(*tm, sb.data_mapping_root_,
mapping_tree_detail::block_traits::ref_counter(space_map::ptr()));
count_btree_blocks(mtree, bc, vc);
}
}
void thin_provisioning::count_space_maps(transaction_manager::ptr tm,
superblock_detail::superblock &sb,
block_counter &bc) {
// Count the metadata space map (no-throw)
try {
persistent_space_map::ptr metadata_sm =
open_metadata_sm(*tm, static_cast<void *>(&sb.metadata_space_map_root_));
metadata_sm->count_metadata(bc);
} catch (std::exception &e) {
cerr << e.what() << endl;
}
// Count the data space map (no-throw)
try {
persistent_space_map::ptr data_sm =
open_disk_sm(*tm, static_cast<void *>(&sb.data_space_map_root_));
data_sm->count_metadata(bc);
} catch (std::exception &e) {
cerr << e.what() << endl;
}
}
void thin_provisioning::count_metadata(transaction_manager::ptr tm,
superblock_detail::superblock &sb,
block_counter &bc,
bool skip_metadata_snap) {
// Count the superblock
bc.inc(superblock_detail::SUPERBLOCK_LOCATION);
count_trees(tm, sb, bc);
// Count the metadata snap, if present
if (!skip_metadata_snap && sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) {
bc.inc(sb.metadata_snap_);
superblock_detail::superblock snap = read_superblock(tm->get_bm(), sb.metadata_snap_);
count_trees(tm, snap, bc);
}
count_trees(tm, sb, bc);
count_space_maps(tm, sb, bc);
}
//----------------------------------------------------------------

24
thin-provisioning/metadata_counter.h Normal file
View File

@ -0,0 +1,24 @@
#ifndef METADATA_COUNTER_H
#define METADATA_COUNTER_H
#include "thin-provisioning/metadata.h"
#include "persistent-data/data-structures/btree_counter.h"
//----------------------------------------------------------------
namespace thin_provisioning {
void count_trees(transaction_manager::ptr tm,
superblock_detail::superblock &sb,
block_counter &bc);
void count_space_maps(transaction_manager::ptr tm,
superblock_detail::superblock &sb,
block_counter &bc);
void count_metadata(transaction_manager::ptr tm,
superblock_detail::superblock &sb,
block_counter &bc,
bool skip_metadata_snap = false);
}
//----------------------------------------------------------------
#endif

13
thin-provisioning/metadata_dumper.cc
View File

@ -228,6 +228,8 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair)
e->begin_superblock("", md->sb_.time_,
md->sb_.trans_id_,
md->sb_.flags_,
md->sb_.version_,
md->sb_.data_block_size_,
nr_data_blocks,
boost::optional<block_address>());
@ -242,3 +244,14 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair)
}
//----------------------------------------------------------------
void
thin_provisioning::metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root) {
mapping_emitter me(e);
single_mapping_tree tree(*md->tm_, subtree_root,
mapping_tree_detail::block_time_ref_counter(md->data_sm_));
walk_mapping_tree(tree, static_cast<mapping_tree_detail::mapping_visitor &>(me),
*mapping_damage_policy(repair));
}
//----------------------------------------------------------------

1
thin-provisioning/metadata_dumper.h
View File

@ -29,6 +29,7 @@ namespace thin_provisioning {
// the dumper to do it's best to recover info. If not set, any
// corruption encountered will cause an exception to be thrown.
void metadata_dump(metadata::ptr md, emitter::ptr e, bool repair);
void metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root);
}
//----------------------------------------------------------------

19
thin-provisioning/restore_emitter.cc
View File

@ -44,6 +44,8 @@ namespace {
virtual void begin_superblock(std::string const &uuid,
uint64_t time,
uint64_t trans_id,
boost::optional<uint32_t> flags,
boost::optional<uint32_t> version,
uint32_t data_block_size,
uint64_t nr_data_blocks,
boost::optional<uint64_t> metadata_snap) {
@ -54,6 +56,8 @@ namespace {
memcpy(&sb.uuid_, uuid.c_str(), std::min(sizeof(sb.uuid_), uuid.length()));
sb.time_ = time;
sb.trans_id_ = trans_id;
sb.flags_ = flags ? *flags : 0;
sb.version_ = version ? *version : 1;
sb.data_block_size_ = data_block_size;
sb.metadata_snap_ = metadata_snap ? *metadata_snap : 0;
md_->data_sm_->extend(nr_data_blocks);
@ -78,10 +82,11 @@ namespace {
if (device_exists(dev))
throw std::runtime_error("Device already exists");
// Add entry to the details tree
uint64_t key[1] = {dev};
device_tree_detail::device_details details = {mapped_blocks, trans_id, (uint32_t)creation_time, (uint32_t)snap_time};
md_->details_->insert(key, details);
// Store the entry of the details tree
current_device_details_.mapped_blocks_ = 0;
current_device_details_.transaction_id_ = trans_id;
current_device_details_.creation_time_ = (uint32_t)creation_time;
current_device_details_.snapshotted_time_ = (uint32_t)snap_time;
current_mapping_ = empty_mapping_->clone();
current_device_ = boost::optional<uint32_t>(dev);
@ -90,6 +95,9 @@ namespace {
virtual void end_device() {
uint64_t key[1] = {*current_device_};
// Add entry to the details tree
md_->details_->insert(key, current_device_details_);
md_->mappings_top_level_->insert(key, current_mapping_->get_root());
md_->mappings_->set_root(md_->mappings_top_level_->get_root()); // FIXME: ugly
@ -130,6 +138,8 @@ namespace {
bt.time_ = time;
current_mapping_->insert(key, bt);
md_->data_sm_->inc(data_block);
current_device_details_.mapped_blocks_ += 1;
}
private:
@ -149,6 +159,7 @@ namespace {
bool in_superblock_;
block_address nr_data_blocks_;
boost::optional<uint32_t> current_device_;
device_tree_detail::device_details current_device_details_;
single_mapping_tree::ptr current_mapping_;
single_mapping_tree::ptr empty_mapping_;
};

14
thin-provisioning/superblock.cc
View File

@ -88,11 +88,23 @@ namespace {
struct sb_validator : public bcache::validator {
virtual void check(void const *raw, block_address location) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(sbd->csum_)) {
ostringstream out;
out << "bad checksum in superblock, wanted " << sum.get_sum();
throw checksum_error(out.str());
}
}
virtual bool check_raw(void const *raw) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(sbd->csum_))
throw checksum_error("bad checksum in superblock");
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {

473
thin-provisioning/thin_ll_dump.cc Normal file
View File

@ -0,0 +1,473 @@
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <getopt.h>
#include <vector>
#include <fstream>
#include "base/indented_stream.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/btree_counter.h"
#include "persistent-data/data-structures/simple_traits.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/metadata_counter.h"
#include "thin-provisioning/commands.h"
#include "version.h"
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
// extracted from btree_damage_visitor.h
template <typename node>
bool check_block_nr(node const &n) {
if (n.get_location() != n.get_block_nr()) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_max_entries(node const &n) {
size_t elt_size = sizeof(uint64_t) + n.get_value_size();
if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) {
return false;
}
if (n.get_max_entries() % 3) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_nr_entries(node const &n, bool is_root) {
if (n.get_nr_entries() > n.get_max_entries()) {
return false;
}
block_address min = n.get_max_entries() / 3;
if (!is_root && (n.get_nr_entries() < min)) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_ordered_keys(node const &n) {
unsigned nr_entries = n.get_nr_entries();
if (nr_entries == 0)
return true; // can only happen if a root node
uint64_t last_key = n.key_at(0);
for (unsigned i = 1; i < nr_entries; i++) {
uint64_t k = n.key_at(i);
if (k <= last_key) {
return false;
}
last_key = k;
}
return true;
}
transaction_manager::ptr
open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(superblock_detail::SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}
}
//---------------------------------------------------------------------------
namespace {
struct node_info {
uint64_t blocknr_;
uint32_t flags_;
uint64_t key_begin_;
uint64_t key_end_;
uint64_t nr_entries_;
uint32_t value_size_;
};
//-------------------------------------------------------------------
struct btree_node_checker {
typedef boost::shared_ptr<btree_node_checker> ptr;
virtual ~btree_node_checker() {}
virtual bool check(node_ref<uint64_traits> &n) = 0;
};
struct unvisited_btree_node_filter: public btree_node_checker {
unvisited_btree_node_filter(block_counter const &bc)
: nv_(create_btree_node_validator()), bc_(bc) {
}
virtual bool check(node_ref<uint64_traits> &n) {
uint32_t flags = to_cpu<uint32_t>(n.raw()->header.flags);
if ((n.get_value_size() == sizeof(mapping_tree_detail::block_traits::disk_type) ||
n.get_value_size() == sizeof(device_tree_detail::device_details_traits::disk_type)) &&
!bc_.get_count(n.get_location()) &&
check_block_nr(n) &&
(((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) ||
(flags & LEAF_NODE)) &&
nv_->check_raw(n.raw()) &&
check_max_entries(n) &&
check_nr_entries(n, true) &&
check_ordered_keys(n))
return true;
return false;
}
bcache::validator::ptr nv_;
block_counter const &bc_;
};
//-------------------------------------------------------------------
void find_btree_nodes(block_manager<>::ptr bm,
block_address begin,
block_address end,
btree_node_checker::ptr checker,
base::run_set<block_address> &found) {
using namespace persistent_data;
for (block_address b = begin; b < end; ++b) {
block_manager<>::read_ref rr = bm->read_lock(b);
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
if (checker->check(n))
found.add(b);
}
}
//-------------------------------------------------------------------
bool first_key_cmp(node_info const &lhs, node_info const &rhs) {
return lhs.key_begin_ < rhs.key_begin_;
}
template <typename ValueTraits>
void convert_to_node_info(node_ref<ValueTraits> const &n, node_info &ni) {
ni.blocknr_ = n.get_location();
ni.flags_ = to_cpu<uint32_t>(n.raw()->header.flags);
if ((ni.nr_entries_ = n.get_nr_entries()) > 0) {
ni.key_begin_ = n.key_at(0);
ni.key_end_ = n.key_at(n.get_nr_entries() - 1);
}
ni.value_size_ = n.get_value_size();
}
void output_node_info(indented_stream &out, node_info const &ni) {
out.indent();
out << "<node blocknr=\"" << ni.blocknr_
<< "\" flags=\"" << ni.flags_
<< "\" key_begin=\"" << ni.key_begin_
<< "\" key_end=\"" << ni.key_end_
<< "\" nr_entries=\"" << ni.nr_entries_
<< "\" value_size=\"" << ni.value_size_
<< "\"/>" << endl;
}
//-------------------------------------------------------------------
class ll_mapping_tree_emitter : public mapping_tree_detail::device_visitor {
public:
ll_mapping_tree_emitter(block_manager<>::ptr bm,
indented_stream &out)
: bm_(bm), out_(out) {
}
void visit(btree_path const &path, block_address tree_root) {
out_.indent();
out_ << "<device dev_id=\"" << path[0] <<"\">" << endl;
out_.inc();
// Do not throw exception. Process the next entry inside the current node.
try {
block_manager<>::read_ref rr = bm_->read_lock(tree_root);
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
node_info ni;
convert_to_node_info(n, ni);
output_node_info(out_, ni);
} catch (std::exception &e) {
cerr << e.what() << endl;
}
out_.dec();
out_.indent();
out_ << "</device>" << endl;
}
private:
block_manager<>::ptr bm_;
indented_stream& out_;
};
//-------------------------------------------------------------------
struct flags {
flags() : use_metadata_snap_(false) {
}
bool use_metadata_snap_;
boost::optional<block_address> metadata_snap_;
boost::optional<block_address> data_mapping_root_;
boost::optional<block_address> device_details_root_;
boost::optional<block_address> scan_begin_;
boost::optional<block_address> scan_end_;
};
int low_level_dump_(string const &input,
std::ostream &output,
flags const &f) {
block_manager<>::ptr bm = open_bm(input, block_manager<>::READ_ONLY);
block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0;
block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks();
// Allow to read superblock at arbitrary location for low-level dump,
// without checking equality between the given metadata_snap and sb.metadata_snap_
superblock_detail::superblock sb = read_superblock(bm, superblock_detail::SUPERBLOCK_LOCATION);
if (f.use_metadata_snap_) {
sb = f.metadata_snap_ ?
read_superblock(bm, *f.metadata_snap_) :
read_superblock(bm, sb.metadata_snap_);
}
// override sb.data_mapping_root_
if (f.data_mapping_root_)
sb.data_mapping_root_ = *f.data_mapping_root_;
// override sb.device_details_root_
if (f.device_details_root_)
sb.device_details_root_ = *f.device_details_root_;
transaction_manager::ptr tm = open_tm(bm);
indented_stream out(output);
out.indent();
out << "<superblock blocknr=\"" << sb.blocknr_
<< "\" data_mapping_root=\"" << sb.data_mapping_root_
<< "\" device_details_root=\"" << sb.device_details_root_
<< "\">" << endl;
out.inc();
// output the top-level data mapping tree
ll_mapping_tree_emitter ll_mte(tm->get_bm(), out);
dev_tree dtree(*tm, sb.data_mapping_root_,
mapping_tree_detail::mtree_traits::ref_counter(tm));
noop_damage_visitor noop_dv;
btree_visit_values(dtree, ll_mte, noop_dv);
out.dec();
out.indent();
out << "</superblock>" << endl;
// find orphans
binary_block_counter bc;
bc.inc(superblock_detail::SUPERBLOCK_LOCATION);
count_metadata(tm, sb, bc, true);
btree_node_checker::ptr filter = btree_node_checker::ptr(
new unvisited_btree_node_filter(bc));
base::run_set<block_address> orphans;
find_btree_nodes(bm, scan_begin, scan_end, filter, orphans);
// sort orphans
std::vector<node_info> nodes;
for (base::run_set<block_address>::const_iterator it = orphans.begin();
it != orphans.end();
++it) {
if (it->begin_ && it->end_) {
for (block_address b = *it->begin_; b < *it->end_; ++b) {
block_manager<>::read_ref rr = bm->read_lock(b);
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
nodes.push_back(node_info());
convert_to_node_info(n, nodes.back());
}
}
}
std::sort(nodes.begin(), nodes.end(), first_key_cmp);
// output orphans
out.indent();
out << "<orphans>" << std::endl;
out.inc();
for (size_t i = 0; i < nodes.size(); ++i)
output_node_info(out, nodes[i]);
out.dec();
out.indent();
out << "</orphans>" << std::endl;
return 0;
}
int low_level_dump(string const &input,
boost::optional<string> output,
flags const &f) {
try {
if (output) {
ofstream out(output->c_str());
low_level_dump_(input, out, f);
} else
low_level_dump_(input, cout, f);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
}
//---------------------------------------------------------------------------
thin_ll_dump_cmd::thin_ll_dump_cmd()
: command("thin_ll_dump")
{
}
void
thin_ll_dump_cmd::usage(ostream &out) const {
out << "Usage: " << get_name() << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-m|--metadata-snap}[block#]" << endl
<< " {-o|--output} <xml file>" << endl
<< " {--begin} <block#>" << endl
<< " {--end} <block#>" << endl
<< " {--data-mapping-root} <block#>" << endl
<< " {--device-details-root} <block#>" << endl
<< " {-V|--version}" << endl;
}
int
thin_ll_dump_cmd::run(int argc, char **argv)
{
const char shortopts[] = "hm:o:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "metadata-snap", optional_argument, NULL, 'm'},
{ "output", required_argument, NULL, 'o'},
{ "version", no_argument, NULL, 'V'},
{ "begin", required_argument, NULL, 1},
{ "end", required_argument, NULL, 2},
{ "data-mapping-root", required_argument, NULL, 3},
{ "device-details-root", required_argument, NULL, 4},
{ NULL, no_argument, NULL, 0 }
};
boost::optional<string> output;
flags f;
char c;
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'm':
f.use_metadata_snap_ = true;
if (optarg) {
try {
f.metadata_snap_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
}
break;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
case 1:
try {
f.scan_begin_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 2:
try {
f.scan_end_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 3:
try {
f.data_mapping_root_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 4:
try {
f.device_details_root_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
default:
usage(cerr);
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr);
return 1;
}
if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) {
cerr << "badly formed region (end <= begin)" << endl;
usage(cerr);
return 1;
}
return low_level_dump(argv[optind], output, f);
}
//---------------------------------------------------------------------------

276
thin-provisioning/thin_ll_restore.cc Normal file
View File

@ -0,0 +1,276 @@
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include "base/xml_utils.h"
#include "metadata_dumper.h"
#include "metadata.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "restore_emitter.h"
#include "xml_format.h"
#include "thin-provisioning/commands.h"
#include "version.h"
#include <fstream>
#include <getopt.h>
#include <iostream>
using namespace persistent_data;
using namespace std;
using namespace thin_provisioning;
using namespace xml_utils;
//----------------------------------------------------------------
namespace {
struct user_data {
block_manager<>::ptr input_bm_;
block_manager<>::ptr output_bm_;
metadata::ptr md_;
XML_Parser parser_;
emitter::ptr emitter_;
};
void open_resources(user_data &ud, attributes const &attr) {
boost::optional<uint64_t> val;
// open the input metadata
// Allow to read superblock at arbitrary location for low-level restore
block_address sb_location = (val = get_opt_attr<uint64_t>(attr, "blocknr")) ?
*val : superblock_detail::SUPERBLOCK_LOCATION;
ud.md_ = metadata::ptr(new metadata(ud.input_bm_, sb_location));
// override superblock::device_details_root_
if ((val = get_opt_attr<uint64_t>(attr, "device_details_root"))) {
ud.md_->sb_.device_details_root_ = *val;
ud.md_->details_ = device_tree::ptr(new device_tree(*ud.md_->tm_, *val,
device_tree_detail::device_details_traits::ref_counter()));
}
// open the output metadata
metadata::ptr new_md(new metadata(ud.output_bm_, metadata::CREATE, 128, 0));
ud.emitter_ = create_restore_emitter(new_md);
}
void parse_superblock(metadata::ptr md, emitter::ptr e, attributes const &attr) {
sm_disk_detail::sm_root_disk const *d =
reinterpret_cast<sm_disk_detail::sm_root_disk const *>(md->sb_.data_space_map_root_);
sm_disk_detail::sm_root v;
sm_disk_detail::sm_root_traits::unpack(*d, v);
e->begin_superblock("", md->sb_.time_,
md->sb_.trans_id_,
md->sb_.flags_,
md->sb_.version_,
md->sb_.data_block_size_,
v.nr_blocks_,
boost::optional<block_address>());
}
void parse_device(metadata::ptr md, emitter::ptr e, attributes const &attr) {
uint32_t dev_id = get_attr<uint32_t>(attr, "dev_id");
device_tree_detail::device_details details;
details.transaction_id_ = 0;
details.creation_time_ = 0;
details.snapshotted_time_ = 0;
device_tree::ptr details_tree;
boost::optional<uint32_t> details_root = get_opt_attr<uint32_t>(attr, "blocknr");
if (details_root)
details_tree = device_tree::ptr(new device_tree(*md->tm_, *details_root,
device_tree_detail::device_details_traits::ref_counter()));
else
details_tree = md->details_;
uint64_t key[1] = {dev_id};
device_tree::maybe_value v;
try {
v = details_tree->lookup(key);
} catch (std::exception &e) {
cerr << "missing device " << dev_id << ": " << e.what() << endl;
}
if (v)
details = *v;
e->begin_device(dev_id,
0,
details.transaction_id_,
details.creation_time_,
details.snapshotted_time_);
}
void parse_node(metadata::ptr md, emitter::ptr e, attributes const &attr) {
metadata_dump_subtree(md, e, true, get_attr<uint32_t>(attr, "blocknr"));
}
void start_tag(void *data, char const *el, char const **attr) {
user_data *ud = static_cast<user_data *>(data);
attributes a;
build_attributes(a, attr);
if (!strcmp(el, "superblock")) {
open_resources(*ud, a);
parse_superblock(ud->md_, ud->emitter_, a);
} else if (!strcmp(el, "device"))
parse_device(ud->md_, ud->emitter_, a);
else if (!strcmp(el, "node"))
parse_node(ud->md_, ud->emitter_, a);
else
throw runtime_error("unknown tag type");
}
void end_tag(void *data, const char *el) {
user_data *ud = static_cast<user_data *>(data);
if (!strcmp(el, "superblock")) {
ud->emitter_->end_superblock();
XML_StopParser(ud->parser_, XML_FALSE); // skip the rest elements
}
else if (!strcmp(el, "device"))
ud->emitter_->end_device();
else if (!strcmp(el, "node"))
;
else
throw runtime_error("unknown tag type");
}
}
//---------------------------------------------------------------------------
namespace {
struct flags {
flags() {
}
};
int low_level_restore_(string const &src_metadata, string const &input,
string const &output, flags const &f) {
user_data ud;
ud.input_bm_ = open_bm(src_metadata, block_manager<>::READ_ONLY);
ud.output_bm_ = open_bm(output, block_manager<>::READ_WRITE);
xml_parser p;
ud.parser_ = p.get_parser();
XML_SetUserData(p.get_parser(), &ud);
XML_SetElementHandler(p.get_parser(), start_tag, end_tag);
bool quiet = true;
p.parse(input, quiet);
return 0;
}
int low_level_restore(string const &src_metadata, string const &input,
string const &output, flags const &f) {
try {
low_level_restore_(src_metadata, input, output, f);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
}
//---------------------------------------------------------------------------
thin_ll_restore_cmd::thin_ll_restore_cmd()
: command("thin_ll_restore")
{
}
void
thin_ll_restore_cmd::usage(ostream &out) const {
out << "Usage: " << get_name() << " [options]" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-E|--source-metadata} <input device or file>" << endl
<< " {-i|--input} <input xml file>" << endl
<< " {-o|--output} <output device or file>" << endl
<< " {-V|--version}" << endl;
}
int
thin_ll_restore_cmd::run(int argc, char **argv) {
string input;
string output;
string input_metadata;
flags f;
char c;
const char shortopts[] = "hi:o:E:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i'},
{ "output", required_argument, NULL, 'o'},
{ "source-metadata", required_argument, NULL, 'E'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'i':
input = optarg;
break;
case 'o':
output = optarg;
break;
case 'E':
input_metadata = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr);
return 1;
}
}
if (argc != optind) {
usage(cerr);
return 1;
}
if (!input_metadata.length() || !input.length() || !output.length()) {
cerr << "No input/output file provided." << endl;
usage(cerr);
return 1;
}
return low_level_restore(input_metadata, input, output, f);
}
//---------------------------------------------------------------------------

417
thin-provisioning/thin_scan.cc Normal file
View File

@ -0,0 +1,417 @@
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <getopt.h>
#include <vector>
#include <fstream>
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/simple_traits.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/superblock.h"
#include "thin-provisioning/commands.h"
#include "version.h"
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
// extracted from btree_damage_visitor.h
template <typename node>
bool check_block_nr(node const &n) {
if (n.get_location() != n.get_block_nr()) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_max_entries(node const &n) {
size_t elt_size = sizeof(uint64_t) + n.get_value_size();
if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) {
return false;
}
if (n.get_max_entries() % 3) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_nr_entries(node const &n, bool is_root) {
if (n.get_nr_entries() > n.get_max_entries()) {
return false;
}
block_address min = n.get_max_entries() / 3;
if (!is_root && (n.get_nr_entries() < min)) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_ordered_keys(node const &n) {
unsigned nr_entries = n.get_nr_entries();
if (nr_entries == 0)
return true; // can only happen if a root node
uint64_t last_key = n.key_at(0);
for (unsigned i = 1; i < nr_entries; i++) {
uint64_t k = n.key_at(i);
if (k <= last_key) {
return false;
}
last_key = k;
}
return true;
}
transaction_manager::ptr
open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(superblock_detail::SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}
}
namespace {
// FIXME: deprecated conversion from string constant to ‘char*’
char const* metadata_block_type_name[] = {
"unknown",
"zero",
"superblock",
"btree_internal",
"btree_leaf",
"btree_unknown",
"index_block",
"bitmap_block"
};
enum metadata_block_type {
UNKNOWN = 0,
ZERO,
SUPERBLOCK,
BTREE_INTERNAL,
BTREE_LEAF,
BTREE_UNKNOWN,
INDEX_BLOCK,
BITMAP_BLOCK
};
struct block_range {
uint64_t begin_;
uint64_t end_; // one-pass-the-end
boost::optional<uint64_t> blocknr_begin_;
metadata_block_type type_;
int64_t ref_count_; // ref_count in metadata space map
size_t value_size_; // btree node only
bool is_valid_; // btree node only
block_range()
: begin_(0), end_(0),
type_(UNKNOWN), ref_count_(-1),
value_size_(0), is_valid_(false)
{
}
block_range(block_range const &rhs)
: begin_(rhs.begin_), end_(rhs.end_),
blocknr_begin_(rhs.blocknr_begin_),
type_(rhs.type_), ref_count_(rhs.ref_count_),
value_size_(rhs.value_size_), is_valid_(rhs.is_valid_)
{
}
};
void output_block_range(block_range const &r, std::ostream &out) {
if (r.end_ <= r.begin_)
return;
if (r.end_ - r.begin_ > 1) {
out << "<range_block type=\"" << metadata_block_type_name[r.type_]
<< "\" location_begin=\"" << r.begin_;
if (r.blocknr_begin_)
out << "\" blocknr_begin=\"" << *r.blocknr_begin_;
out << "\" length=\"" << r.end_ - r.begin_
<< "\" ref_count=\"" << r.ref_count_
<< "\" is_valid=\"" << r.is_valid_;
} else {
out << "<single_block type=\"" << metadata_block_type_name[r.type_]
<< "\" location=\"" << r.begin_;
if (r.blocknr_begin_)
out << "\" blocknr=\"" << *r.blocknr_begin_;
out << "\" ref_count=\"" << r.ref_count_
<< "\" is_valid=\"" << r.is_valid_;
}
if (r.type_ == BTREE_INTERNAL || r.type_ == BTREE_LEAF || r.type_ == BTREE_UNKNOWN) {
out << "\" value_size=\"" << r.value_size_ << "\"/>" << endl;
} else
out << "\"/>" << endl;
}
//-------------------------------------------------------------------
struct flags {
flags() {
}
boost::optional<block_address> scan_begin_;
boost::optional<block_address> scan_end_;
};
int scan_metadata_(string const &input,
std::ostream &out,
flags const &f) {
using namespace persistent_data;
using namespace thin_provisioning;
using namespace sm_disk_detail;
block_manager<>::ptr bm;
bm = open_bm(input, block_manager<>::READ_ONLY);
block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0;
block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks();
const std::vector<char> zeros(MD_BLOCK_SIZE, 0);
// try to open metadata space-map (it's okay to fail)
// note: transaction_manager and space_map must be in the same scope
transaction_manager::ptr tm;
checked_space_map::ptr metadata_sm;
try {
superblock_detail::superblock sb = read_superblock(bm);
tm = open_tm(bm);
metadata_sm = open_metadata_sm(*tm, &sb.metadata_space_map_root_);
tm->set_sm(metadata_sm);
} catch (std::exception &e) {
cerr << e.what() << endl;
}
block_range curr_range;
block_range run_range;
bcache::validator::ptr sv = superblock_validator();
bcache::validator::ptr nv = create_btree_node_validator();
bcache::validator::ptr iv = index_validator();
bcache::validator::ptr bv = bitmap_validator();
for (block_address b = scan_begin; b < scan_end; ++b) {
block_manager<>::read_ref rr = bm->read_lock(b);
curr_range.begin_ = b;
curr_range.end_ = b + 1;
curr_range.blocknr_begin_ = boost::none;
curr_range.type_ = UNKNOWN;
curr_range.is_valid_ = false;
if (!memcmp(rr.data(), zeros.data(), MD_BLOCK_SIZE))
curr_range.type_ = ZERO;
if (curr_range.type_ == UNKNOWN && sv->check_raw(rr.data())) {
curr_range.type_ = SUPERBLOCK;
curr_range.is_valid_ = true;
}
if (curr_range.type_ == UNKNOWN && nv->check_raw(rr.data())) {
// note: check_raw() doesn't check node_header::blocknr_
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
uint32_t flags = to_cpu<uint32_t>(n.raw()->header.flags);
if ((flags & INTERNAL_NODE) && !(flags & LEAF_NODE))
curr_range.type_ = BTREE_INTERNAL;
else if (flags & LEAF_NODE)
curr_range.type_ = BTREE_LEAF;
else
curr_range.type_ = BTREE_UNKNOWN;
if (curr_range.type_ != BTREE_UNKNOWN &&
check_block_nr(n) &&
check_max_entries(n) &&
check_nr_entries(n, true) &&
check_ordered_keys(n))
curr_range.is_valid_ = true;
else
curr_range.is_valid_ = false;
curr_range.blocknr_begin_ = n.get_block_nr();
curr_range.value_size_ = n.get_value_size();
}
if (curr_range.type_ == UNKNOWN && bv->check_raw(rr.data())) {
curr_range.type_ = BITMAP_BLOCK;
bitmap_header const *data = reinterpret_cast<bitmap_header const *>(rr.data());
curr_range.blocknr_begin_ = to_cpu<uint64_t>(data->blocknr);
curr_range.is_valid_ = (to_cpu<uint64_t>(data->blocknr) == b) ? true : false;
}
if (curr_range.type_ == UNKNOWN && iv->check_raw(rr.data())) {
curr_range.type_ = INDEX_BLOCK;
metadata_index const *mi = reinterpret_cast<metadata_index const *>(rr.data());
curr_range.blocknr_begin_ = to_cpu<uint64_t>(mi->blocknr_);
curr_range.is_valid_ = (to_cpu<uint64_t>(mi->blocknr_) == b) ? true : false;
}
try {
curr_range.ref_count_ = metadata_sm ?
static_cast<int64_t>(metadata_sm->get_count(b)) : -1;
} catch (std::exception &e) {
curr_range.ref_count_ = -1;
}
// output the current block
if (run_range.end_ == 0)
run_range = curr_range;
else if (((!curr_range.blocknr_begin_ && !run_range.blocknr_begin_) ||
(curr_range.blocknr_begin_ && run_range.blocknr_begin_ &&
*curr_range.blocknr_begin_ == *run_range.blocknr_begin_ + (run_range.end_ - run_range.begin_))) &&
curr_range.type_ == run_range.type_ &&
curr_range.ref_count_ == run_range.ref_count_ &&
curr_range.value_size_ == run_range.value_size_ &&
curr_range.is_valid_ == run_range.is_valid_) {
++run_range.end_;
} else {
output_block_range(run_range, out);
run_range = curr_range;
}
}
// output the last run
output_block_range(run_range, out);
return 0;
}
int scan_metadata(string const &input,
boost::optional<string> output,
flags const &f) {
try {
if (output) {
std::ofstream out(output->c_str());
scan_metadata_(input, out, f);
} else
scan_metadata_(input, cout, f);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
}
//---------------------------------------------------------------------------
thin_scan_cmd::thin_scan_cmd()
: command("thin_scan")
{
}
void
thin_scan_cmd::usage(std::ostream &out) const {
out << "Usage: " << get_name() << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-o|--output} <xml file>" << endl
<< " {--begin} <block#>" << endl
<< " {--end} <block#>" << endl
<< " {-V|--version}" << endl;
}
int
thin_scan_cmd::run(int argc, char **argv)
{
const char shortopts[] = "ho:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "output", required_argument, NULL, 'o'},
{ "version", no_argument, NULL, 'V'},
{ "begin", required_argument, NULL, 1},
{ "end", required_argument, NULL, 2},
{ NULL, no_argument, NULL, 0 }
};
boost::optional<string> output;
flags f;
char c;
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
case 1:
try {
f.scan_begin_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 2:
try {
f.scan_end_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
default:
usage(cerr);
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr);
return 1;
}
if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) {
cerr << "badly formed region (end <= begin)" << endl;
return 1;
}
return scan_metadata(argv[optind], output, f);
}
//---------------------------------------------------------------------------

6
thin-provisioning/xml_format.cc
View File

@ -50,6 +50,8 @@ namespace {
void begin_superblock(string const &uuid,
uint64_t time,
uint64_t trans_id,
boost::optional<uint32_t> flags,
boost::optional<uint32_t> version,
uint32_t data_block_size,
uint64_t nr_data_blocks,
boost::optional<uint64_t> metadata_snap) {
@ -57,6 +59,8 @@ namespace {
out_ << "<superblock uuid=\"" << uuid << "\""
<< " time=\"" << time << "\""
<< " transaction=\"" << trans_id << "\""
<< " flags=\"" << (flags ? *flags : 0) << "\""
<< " version=\"" << (version ? *version : 1) << "\""
<< " data_block_size=\"" << data_block_size << "\""
<< " nr_data_blocks=\"" << nr_data_blocks;
@ -140,6 +144,8 @@ namespace {
e->begin_superblock(get_attr<string>(attr, "uuid"),
get_attr<uint64_t>(attr, "time"),
get_attr<uint64_t>(attr, "transaction"),
get_opt_attr<uint32_t>(attr, "flags"),
get_opt_attr<uint32_t>(attr, "version"),
get_attr<uint32_t>(attr, "data_block_size"),
get_attr<uint64_t>(attr, "nr_data_blocks"),
get_opt_attr<uint64_t>(attr, "metadata_snap"));

8
unit-tests/block_t.cc
View File

@ -44,6 +44,14 @@ namespace {
throw runtime_error("validator check zero");
}
virtual bool check_raw(void const *raw) const {
unsigned char const *data = reinterpret_cast<unsigned char const *>(raw);
for (unsigned b = 0; b < BlockSize; b++)
if (data[b] != 0)
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
unsigned char *data = reinterpret_cast<unsigned char *>(raw);
for (unsigned b = 0; b < BlockSize; b++)