Merge pull request #144 from mingnus/thin-generate-metadata-wip

Enhance thin_generate_mappings, and introduce thin_generate_damage
This commit is contained in:
Joe Thornber 2020-07-27 15:55:22 +01:00 committed by GitHub
commit 75ba654b7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 420 additions and 55 deletions

View File

@ -123,11 +123,13 @@ SOURCE=\
thin-provisioning/xml_format.cc thin-provisioning/xml_format.cc
DEVTOOLS_SOURCE=\ DEVTOOLS_SOURCE=\
thin-provisioning/damage_generator.cc \
thin-provisioning/thin_journal.cc \ thin-provisioning/thin_journal.cc \
thin-provisioning/thin_journal_check.cc \ thin-provisioning/thin_journal_check.cc \
thin-provisioning/thin_ll_dump.cc \ thin-provisioning/thin_ll_dump.cc \
thin-provisioning/thin_ll_restore.cc \ thin-provisioning/thin_ll_restore.cc \
thin-provisioning/thin_show_duplicates.cc \ thin-provisioning/thin_show_duplicates.cc \
thin-provisioning/thin_generate_damage.cc \
thin-provisioning/thin_generate_metadata.cc \ thin-provisioning/thin_generate_metadata.cc \
thin-provisioning/thin_generate_mappings.cc \ thin-provisioning/thin_generate_mappings.cc \
thin-provisioning/variable_chunk_stream.cc \ thin-provisioning/variable_chunk_stream.cc \

View File

@ -1,4 +1,5 @@
#include "base/io_generator.h" #include "base/io_generator.h"
#include "persistent-data/run_set.h"
#include <stdexcept> #include <stdexcept>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
@ -25,61 +26,146 @@ namespace {
//-------------------------------- //--------------------------------
class offset_generator { class sequence_generator {
public: public:
typedef std::shared_ptr<offset_generator> ptr; typedef std::shared_ptr<sequence_generator> ptr;
virtual base::sector_t next_offset() = 0; virtual uint64_t next() = 0;
}; };
class sequential_offset_generator: public offset_generator { // - The maximum generated value is not greater than (begin+size)
// - The generated values are not aligned to the step, if the begin
// value is not aligned.
class forward_sequence_generator: public sequence_generator {
public: public:
sequential_offset_generator(base::sector_t offset, forward_sequence_generator(uint64_t begin,
base::sector_t size, uint64_t size,
base::sector_t block_size) uint64_t step)
: block_size_(block_size), : begin_(begin),
begin_(offset), end_(begin + (size / step) * step),
end_(offset + size), step_(step),
current_(offset) { current_(begin),
if (size < block_size) rounds_(0) {
throw std::runtime_error("size must be greater than block_size"); if (size < step)
throw std::runtime_error("size must be greater than the step");
} }
base::sector_t next_offset() { forward_sequence_generator()
sector_t r = current_; : begin_(0), end_(1), step_(1),
current_ += block_size_; current_(0), rounds_(0) {
if (current_ > end_) { }
uint64_t next() {
uint64_t r = current_;
current_ += step_;
if (current_ >= end_) {
current_ = begin_; current_ = begin_;
return begin_; ++rounds_;
} }
return r; return r;
} }
void reset(uint64_t begin, uint64_t size, uint64_t step = 1) {
begin_ = begin;
end_ = begin + (size / step) * step;
step_ = step;
current_ = begin;
rounds_ = 0;
}
uint64_t get_rounds() {
return rounds_;
}
private: private:
unsigned block_size_; uint64_t begin_;
base::sector_t begin_; uint64_t end_;
base::sector_t end_; uint64_t step_;
base::sector_t current_; uint64_t current_;
uint64_t rounds_;
}; };
class random_offset_generator: public offset_generator { // - The maximum generated value is not greater than (begin+size)
// - The generated values are not aligned to the step, if the begin
// value is not aligned.
class random_sequence_generator: public sequence_generator {
public: public:
random_offset_generator(sector_t offset, // TODO: load random seeds
sector_t size, random_sequence_generator(uint64_t begin,
sector_t block_size) uint64_t size,
: block_begin_(offset / block_size), uint64_t step,
nr_blocks_(size / block_size), unsigned seq_nr = 1)
block_size_(block_size) { : begin_(begin),
nr_steps_(size / step),
step_(step),
max_forward_steps_(seq_nr),
nr_generated_(0)
{
if (!max_forward_steps_ || max_forward_steps_ > nr_steps_)
throw std::runtime_error("invalid number of forward steps");
if (max_forward_steps_ > 1)
reset_forward_generator();
} }
sector_t next_offset() { uint64_t next() {
return ((std::rand() % nr_blocks_) + block_begin_) * block_size_; // FIXME: eliminate if-else
uint64_t step_idx = (max_forward_steps_ > 1) ?
next_forward_step() : next_random_step();
rand_map_.add(step_idx);
++nr_generated_;
// wrap-around
if (nr_generated_ == nr_steps_) {
rand_map_.clear();
nr_generated_ = 0;
}
return begin_ + step_idx * step_;
} }
private: private:
uint64_t block_begin_; void reset_forward_generator() {
uint64_t nr_blocks_; uint64_t begin = next_random_step();
unsigned block_size_; unsigned seq_nr = (std::rand() % max_forward_steps_) + 1;
forward_gen_.reset(begin, seq_nr);
}
uint64_t next_forward_step() {
uint64_t step_idx;
bool found = true;
while (found) {
step_idx = forward_gen_.next();
found = rand_map_.member(step_idx);
if (found || forward_gen_.get_rounds())
reset_forward_generator();
}
return step_idx;
}
uint64_t next_random_step() const {
uint64_t step_idx;
bool found = true;
while (found) {
step_idx = std::rand() % nr_steps_;
found = rand_map_.member(step_idx);
}
return step_idx;
}
uint64_t begin_;
uint64_t nr_steps_;
uint64_t step_;
unsigned max_forward_steps_;
base::run_set<uint64_t> rand_map_;
uint64_t nr_generated_;
forward_sequence_generator forward_gen_;
}; };
//-------------------------------- //--------------------------------
@ -120,13 +206,13 @@ namespace {
virtual bool next(base::io &next_io); virtual bool next(base::io &next_io);
private: private:
offset_generator::ptr sequence_generator::ptr
create_offset_generator(io_generator_options const &opts); create_offset_generator(io_generator_options const &opts);
op_generator::ptr op_generator::ptr
create_op_generator(io_generator_options const &opts); create_op_generator(io_generator_options const &opts);
offset_generator::ptr offset_gen_; sequence_generator::ptr offset_gen_;
op_generator::ptr op_gen_; op_generator::ptr op_gen_;
sector_t block_size_; sector_t block_size_;
size_t io_size_finished_; size_t io_size_finished_;
@ -146,7 +232,7 @@ namespace {
return false; return false;
next_io.op_ = op_gen_->next_op(); next_io.op_ = op_gen_->next_op();
next_io.sector_ = offset_gen_->next_offset(); next_io.sector_ = offset_gen_->next();
next_io.size_ = block_size_; next_io.size_ = block_size_;
io_size_finished_ += block_size_; io_size_finished_ += block_size_;
@ -154,18 +240,19 @@ namespace {
return true; return true;
} }
offset_generator::ptr sequence_generator::ptr
base_io_generator::create_offset_generator(io_generator_options const &opts) { base_io_generator::create_offset_generator(io_generator_options const &opts) {
if (opts.pattern_.is_random()) sequence_generator *gen;
return offset_generator::ptr(
new random_offset_generator(opts.offset_,
opts.size_,
opts.block_size_));
return offset_generator::ptr( if (opts.pattern_.is_random())
new sequential_offset_generator(opts.offset_, gen = new random_sequence_generator(opts.offset_,
opts.size_, opts.size_, opts.block_size_,
opts.block_size_)); opts.nr_seq_blocks_);
else
gen = new forward_sequence_generator(opts.offset_,
opts.size_, opts.block_size_);
return sequence_generator::ptr(gen);
} }
op_generator::ptr op_generator::ptr

View File

@ -36,6 +36,7 @@ namespace base {
sector_t block_size_; sector_t block_size_;
sector_t size_; sector_t size_;
sector_t io_size_; sector_t io_size_;
unsigned nr_seq_blocks_;
}; };
class io_generator { class io_generator {

View File

@ -22,6 +22,7 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_ll_dump_cmd())); app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_restore_cmd())); app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
app.add_cmd(command::ptr(new thin_scan_cmd())); app.add_cmd(command::ptr(new thin_scan_cmd()));
app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
app.add_cmd(command::ptr(new thin_generate_metadata_cmd())); app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
app.add_cmd(command::ptr(new thin_generate_mappings_cmd())); app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
app.add_cmd(command::ptr(new thin_show_duplicates_cmd())); app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));

View File

@ -103,6 +103,13 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv); virtual int run(int argc, char **argv);
}; };
class thin_generate_damage_cmd : public base::command {
public:
thin_generate_damage_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_generate_metadata_cmd : public base::command { class thin_generate_metadata_cmd : public base::command {
public: public:
thin_generate_metadata_cmd(); thin_generate_metadata_cmd();

View File

@ -0,0 +1,66 @@
#include "damage_generator.h"
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
void find_blocks(space_map::ptr sm,
block_address nr_blocks,
ref_t expected,
std::set<block_address> &found) {
block_address sm_size = sm->get_nr_blocks();
base::run_set<block_address> visited;
block_address nr_visited = 0;
srand(time(NULL));
while (nr_blocks) {
if (nr_visited == sm_size)
break;
block_address b = rand() % sm_size;
if (visited.member(b))
continue;
ref_t c = sm->get_count(b);
if (c == expected) {
found.insert(b);
--nr_blocks;
}
visited.add(b);
++nr_visited;
}
if (nr_blocks) {
ostringstream out;
out << "cannot find " << (nr_blocks + found.size())
<< " blocks of ref-count " << expected;
throw runtime_error(out.str());
}
}
}
//----------------------------------------------------------------
damage_generator::damage_generator(block_manager::ptr bm)
{
md_ = metadata::ptr(new metadata(bm, true));
}
void damage_generator::commit()
{
md_->commit();
}
void damage_generator::create_metadata_leaks(block_address nr_leaks,
ref_t expected, ref_t actual)
{
std::set<block_address> leaks;
find_blocks(md_->metadata_sm_, nr_leaks, expected, leaks);
for (auto const &b : leaks)
md_->metadata_sm_->set_count(b, actual);
}
//----------------------------------------------------------------

View File

@ -0,0 +1,22 @@
#ifndef METADATA_DAMAGE_GENERATOR_H
#define METADATA_DAMAGE_GENERATOR_H
#include "metadata.h"
//----------------------------------------------------------------
class damage_generator {
public:
typedef std::shared_ptr<damage_generator> ptr;
damage_generator(block_manager::ptr bm);
void commit();
void create_metadata_leaks(block_address nr_leaks, ref_t expected, ref_t actual);
private:
thin_provisioning::metadata::ptr md_;
};
//----------------------------------------------------------------
#endif

View File

@ -0,0 +1,144 @@
#include "base/output_file_requirements.h"
#include "persistent-data/file_utils.h"
#include "thin-provisioning/commands.h"
#include "thin-provisioning/damage_generator.h"
#include "version.h"
#include <getopt.h>
#include <unistd.h>
using namespace persistent_data;
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
struct flags {
enum damage_operations {
DAMAGE_OP_NONE,
DAMAGE_OP_CREATE_METADATA_LEAKS,
DAMAGE_OP_LAST
};
flags()
: op(DAMAGE_OP_NONE),
nr_blocks(0),
expected_rc(0),
actual_rc(0) {
}
bool check_conformance();
damage_operations op;
string output;
block_address nr_blocks;
ref_t expected_rc;
ref_t actual_rc;
};
bool flags::check_conformance() {
if (op == DAMAGE_OP_NONE || op >= DAMAGE_OP_LAST) {
cerr << "Invalid operation." << endl;
return false;
}
if (!output.size()) {
cerr << "No output file provided." << endl;
return false;
}
if (!nr_blocks) {
cerr << "Invalid number of blocks" << endl;
return false;
}
if (op == DAMAGE_OP_CREATE_METADATA_LEAKS &&
expected_rc == actual_rc) {
cerr << "Invalid reference count parameters" << endl;
return false;
}
check_output_file_requirements(output);
return true;
}
int generate_damage(flags const &fs) {
block_manager::ptr bm = open_bm(fs.output, block_manager::READ_WRITE);
damage_generator::ptr gen = damage_generator::ptr(new damage_generator(bm));
switch (fs.op) {
case flags::DAMAGE_OP_CREATE_METADATA_LEAKS:
gen->create_metadata_leaks(fs.nr_blocks, fs.expected_rc, fs.actual_rc);
break;
default:
break;
}
gen->commit();
return 0;
}
}
//----------------------------------------------------------------
thin_generate_damage_cmd::thin_generate_damage_cmd()
: command("thin_generate_damage")
{
}
void
thin_generate_damage_cmd::usage(std::ostream &out) const
{
}
int
thin_generate_damage_cmd::run(int argc, char **argv)
{
int c;
struct flags fs;
const char *shortopts = "ho:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "create-metadata-leaks", no_argument, NULL, 1 },
{ "nr-blocks", required_argument, NULL, 1001 },
{ "expected", required_argument, NULL, 1002 },
{ "actual", required_argument, NULL, 1003 },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
break;
case 'o':
fs.output = optarg;
break;
case 1:
fs.op = flags::DAMAGE_OP_CREATE_METADATA_LEAKS;
break;
case 1001:
fs.nr_blocks = parse_uint64(optarg, "nr_blocks");
break;
case 1002:
fs.expected_rc = parse_uint64(optarg, "expected");
break;
case 1003:
fs.actual_rc = parse_uint64(optarg, "actual");
break;
}
}
if (!fs.check_conformance()) {
usage(cerr);
return 1;
}
return generate_damage(fs);
}
//----------------------------------------------------------------

View File

@ -47,6 +47,7 @@ namespace {
base::sector_t offset; base::sector_t offset;
boost::optional<base::sector_t> size; boost::optional<base::sector_t> size;
boost::optional<base::sector_t> io_size; boost::optional<base::sector_t> io_size;
boost::optional<unsigned> nr_seq_blocks;
}; };
bool flags::check_conformance() { bool flags::check_conformance() {
@ -65,6 +66,14 @@ namespace {
return false; return false;
} }
if (nr_seq_blocks) {
if (!pattern.is_random()) {
cerr << "Cannot specify the sequence size"
" while doing non-random IO" << endl;
return false;
}
}
check_output_file_requirements(*output); check_output_file_requirements(*output);
return true; return true;
@ -90,6 +99,7 @@ namespace {
opts.offset_ = fs.offset; opts.offset_ = fs.offset;
opts.size_ = *fs.size; opts.size_ = *fs.size;
opts.io_size_ = !fs.io_size ? *fs.size : *fs.io_size; opts.io_size_ = !fs.io_size ? *fs.size : *fs.io_size;
opts.nr_seq_blocks_ = !fs.nr_seq_blocks ? 1 : *fs.nr_seq_blocks;
io_generator::ptr gen = create_io_generator(opts); io_generator::ptr gen = create_io_generator(opts);
base::io io; base::io io;
@ -133,6 +143,7 @@ thin_generate_mappings_cmd::usage(std::ostream &out) const
<< " {--io-size} <io-size in sectors>\n" << " {--io-size} <io-size in sectors>\n"
<< " {--rw write|trim|randwrite|randtrim|randtw}\n" << " {--rw write|trim|randwrite|randtrim|randtw}\n"
<< " {--size} <size in sectors>\n" << " {--size} <size in sectors>\n"
<< " {--seq-nr} <max nr. of sequential ios>\n"
<< " {-V|--version}" << endl; << " {-V|--version}" << endl;
} }
@ -141,7 +152,7 @@ thin_generate_mappings_cmd::run(int argc, char **argv)
{ {
int c; int c;
struct flags fs; struct flags fs;
const char *shortopts = "hi:o:qV"; const char *shortopts = "ho:V";
const struct option longopts[] = { const struct option longopts[] = {
{ "help", no_argument, NULL, 'h' }, { "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' }, { "output", required_argument, NULL, 'o' },
@ -150,6 +161,7 @@ thin_generate_mappings_cmd::run(int argc, char **argv)
{ "offset", required_argument, NULL, 3 }, { "offset", required_argument, NULL, 3 },
{ "size", required_argument, NULL, 4 }, { "size", required_argument, NULL, 4 },
{ "io-size", required_argument, NULL, 5 }, { "io-size", required_argument, NULL, 5 },
{ "seq-nr", required_argument, NULL, 6 },
{ "version", no_argument, NULL, 'V' }, { "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 } { NULL, no_argument, NULL, 0 }
}; };
@ -184,6 +196,10 @@ thin_generate_mappings_cmd::run(int argc, char **argv)
fs.io_size = parse_uint64(optarg, "io_size"); fs.io_size = parse_uint64(optarg, "io_size");
break; break;
case 6:
fs.nr_seq_blocks = parse_uint64(optarg, "seq_nr");
break;
case 'V': case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl; cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0; return 0;

View File

@ -43,6 +43,7 @@ namespace {
METADATA_OP_SET_TRANSACTION_ID, METADATA_OP_SET_TRANSACTION_ID,
METADATA_OP_RESERVE_METADATA_SNAP, METADATA_OP_RESERVE_METADATA_SNAP,
METADATA_OP_RELEASE_METADATA_SNAP, METADATA_OP_RELEASE_METADATA_SNAP,
METADATA_OP_SET_NEEDS_CHECK,
METADATA_OP_LAST METADATA_OP_LAST
}; };
@ -133,6 +134,9 @@ namespace {
case flags::METADATA_OP_RELEASE_METADATA_SNAP: case flags::METADATA_OP_RELEASE_METADATA_SNAP:
pool->release_metadata_snap(); pool->release_metadata_snap();
break; break;
case flags::METADATA_OP_SET_NEEDS_CHECK:
pool->set_needs_check();
break;
default: default:
break; break;
} }
@ -163,6 +167,7 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const
<< " {--reserve-metadata-snap}\n" << " {--reserve-metadata-snap}\n"
<< " {--release-metadata-snap}\n" << " {--release-metadata-snap}\n"
<< " {--set-transaction-id} <tid>\n" << " {--set-transaction-id} <tid>\n"
<< " {--set-needs-check}\n"
<< " {--data-block-size} <block size>\n" << " {--data-block-size} <block size>\n"
<< " {--nr-data-blocks} <nr>\n" << " {--nr-data-blocks} <nr>\n"
<< " {--origin} <origin-id>\n" << " {--origin} <origin-id>\n"
@ -175,7 +180,7 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
{ {
int c; int c;
struct flags fs; struct flags fs;
const char *shortopts = "hi:o:qV"; const char *shortopts = "ho:V";
const struct option longopts[] = { const struct option longopts[] = {
{ "help", no_argument, NULL, 'h' }, { "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' }, { "output", required_argument, NULL, 'o' },
@ -187,9 +192,10 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
{ "set-transaction-id", required_argument, NULL, 6 }, { "set-transaction-id", required_argument, NULL, 6 },
{ "reserve-metadata-snap", no_argument, NULL, 7 }, { "reserve-metadata-snap", no_argument, NULL, 7 },
{ "release-metadata-snap", no_argument, NULL, 8 }, { "release-metadata-snap", no_argument, NULL, 8 },
{ "data-block-size", required_argument, NULL, 101 }, { "set-needs-check", no_argument, NULL, 9 },
{ "nr-data-blocks", required_argument, NULL, 102 }, { "data-block-size", required_argument, NULL, 1001 },
{ "origin", required_argument, NULL, 401 }, { "nr-data-blocks", required_argument, NULL, 1002 },
{ "origin", required_argument, NULL, 4001 },
{ "version", no_argument, NULL, 'V' }, { "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 } { NULL, no_argument, NULL, 0 }
}; };
@ -240,15 +246,19 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
fs.op = flags::METADATA_OP_RELEASE_METADATA_SNAP; fs.op = flags::METADATA_OP_RELEASE_METADATA_SNAP;
break; break;
case 101: case 9:
fs.op = flags::METADATA_OP_SET_NEEDS_CHECK;
break;
case 1001:
fs.data_block_size = parse_uint64(optarg, "data block size"); fs.data_block_size = parse_uint64(optarg, "data block size");
break; break;
case 102: case 1002:
fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks"); fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
break; break;
case 401: case 4001:
fs.origin = parse_uint64(optarg, "origin"); fs.origin = parse_uint64(optarg, "origin");
break; break;

View File

@ -408,6 +408,12 @@ thin_pool::write_changed_details()
} }
} }
void
thin_pool::set_needs_check()
{
md_->sb_.set_needs_check_flag(true);
}
//---------------------------------------------------------------- //----------------------------------------------------------------
void void

View File

@ -99,6 +99,9 @@ namespace thin_provisioning {
thin::ptr open_thin(thin_dev_t); thin::ptr open_thin(thin_dev_t);
void close_thin(thin::ptr td); void close_thin(thin::ptr td);
// updates the superblock
void set_needs_check();
private: private:
friend class thin; friend class thin;
typedef std::map<thin_dev_t, thin::ptr> device_map; typedef std::map<thin_dev_t, thin::ptr> device_map;