emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #136 from mingnus/thin-generate-metadata-wip

Browse Source 
[thin_generate_metadata] Introduce the new transactional metadata generate tool
This commit is contained in:
Joe Thornber 2020-05-28 10:47:04 +01:00 committed by GitHub
commit 774111b34d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 144 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
persistent-data/space-maps/recursive.cc
View File

@ -97,6 +97,12 @@ namespace {
add_op(b, block_op(SET, c));
else {
recursing_lock lock(*this);
// the inner set_count may trigger a find_free,
// so it's important we update the allocated
// blocks list before calling.
allocated_blocks_.add(b, b + 1);
return sm_->set_count(b, c);
}
}
@ -111,6 +117,12 @@ namespace {
add_op(b, block_op(INC, count));
else {
recursing_lock lock(*this);
// the inner inc() may trigger a find_free,
// so it's important we update the allocated
// blocks list before calling.
allocated_blocks_.add(b, b + 1);
return sm_->inc(b, count);
}
}
@ -200,9 +212,11 @@ namespace {
void flush_ops_() {
recursing_lock lock(*this);
for (auto const &p : ops_) {
block_address b = p.first;
auto const &op = p.second;
while (!ops_.empty()) {
auto p = ops_.begin();
block_address b = p->first;
auto op = p->second;
ops_.erase(p);
switch (op.op_) {
case INC:
@ -218,7 +232,6 @@ namespace {
}
}
ops_.clear();
allocated_blocks_.clear();
}

213
thin-provisioning/thin_generate_metadata.cc
View File

@ -19,22 +19,11 @@
#include "base/output_file_requirements.h"
#include "persistent-data/file_utils.h"
#include "thin-provisioning/commands.h"
#include "thin-provisioning/emitter.h"
#include "thin-provisioning/human_readable_format.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/restore_emitter.h"
#include "thin-provisioning/xml_format.h"
#include "version.h"
#include <boost/optional.hpp>
#include <fstream>
#include <getopt.h>
#include <iostream>
#include <libgen.h>
#include <linux/fs.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
using namespace boost;
@ -46,76 +35,128 @@ using namespace thin_provisioning;
namespace {
struct flags {
enum metadata_operations {
METADATA_OP_NONE,
METADATA_OP_FORMAT,
METADATA_OP_OPEN,
METADATA_OP_CREATE_THIN,
METADATA_OP_LAST
};
flags()
: data_block_size(128),
nr_data_blocks(10240),
nr_thins(1),
blocks_per_thin(1024),
run_lengths(1024) {
: op(METADATA_OP_NONE),
data_block_size(128),
nr_data_blocks(10240)
{
}
block_address data_block_size;
bool check_conformance();
metadata_operations op;
sector_t data_block_size;
block_address nr_data_blocks;
unsigned nr_thins;
block_address blocks_per_thin;
block_address run_lengths;
optional<uint64_t> dev_id;
optional<string> output;
};
// This is how we stir in some entropy. It mixes up the data
// device.
class shuffler {
public:
shuffler(block_address nr_blocks, unsigned run_lengths)
: nr_blocks_(nr_blocks / run_lengths),
run_lengths_(run_lengths) {
// FIXME: modulize the conditions
bool flags::check_conformance() {
if (op == METADATA_OP_NONE || op >= METADATA_OP_LAST) {
cerr << "Invalid operation." << endl;
return false;
}
block_address map(block_address b) const {
return reverse(b / run_lengths_) + (b % run_lengths_);
if (!output) {
cerr << "No output file provided." << endl;
return false;
} else
check_output_file_requirements(*output);
if (op == METADATA_OP_CREATE_THIN && !dev_id) {
cerr << "no device id provided." << endl;
return false;
}
private:
block_address reverse(block_address b) const {
return nr_blocks_ - b - 1ull;
}
block_address nr_blocks_;
block_address run_lengths_;
};
void generate_device(emitter::ptr e, shuffler const &s, uint32_t dev_id,
block_address nr_blocks, block_address base) {
e->begin_device(dev_id, nr_blocks, 0, 0, 0);
for (unsigned b = 0; b < nr_blocks; b++)
e->single_map(b, s.map(base + b), 0);
e->end_device();
return true;
}
void generate_metadata(flags const &fs, emitter::ptr e) {
e->begin_superblock("fake metadata", 0, 0, optional<uint32_t>(), optional<uint32_t>(),
fs.data_block_size, fs.nr_data_blocks, optional<uint64_t>());
//--------------------------------
shuffler s(fs.nr_data_blocks, fs.run_lengths);
for (unsigned i = 0; i < fs.nr_thins; i++)
generate_device(e, s, i, fs.blocks_per_thin, i * fs.blocks_per_thin);
e->end_superblock();
single_mapping_tree::ptr new_mapping_tree(metadata::ptr md) {
return single_mapping_tree::ptr(
new single_mapping_tree(*md->tm_,
mapping_tree_detail::block_time_ref_counter(md->data_sm_)));
}
int create_metadata(flags const &fs) {
try {
// The block size gets updated by the restorer.
block_manager::ptr bm(open_bm(*fs.output, block_manager::READ_WRITE));
metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0));
emitter::ptr restorer = create_restore_emitter(md);
bool is_device_exists(metadata::ptr md, uint64_t dev_id) {
uint64_t key[1] = {dev_id};
generate_metadata(fs, restorer);
device_tree::maybe_value v1 = md->details_->lookup(key);
if (v1)
return true;
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
dev_tree::maybe_value v2 = md->mappings_top_level_->lookup(key);
if (v2)
return true;
return false;
}
//--------------------------------
metadata::ptr format_metadata(block_manager::ptr bm,
sector_t data_block_size,
block_address nr_data_blocks) {
metadata::ptr md(new metadata(bm,
metadata::CREATE,
data_block_size,
nr_data_blocks));
md->commit();
return md;
}
metadata::ptr open_metadata(block_manager::ptr bm) {
metadata::ptr md(new metadata(bm, true));
return md;
}
void create_thin(metadata::ptr md, uint64_t dev_id) {
uint64_t key[1] = {dev_id};
if (is_device_exists(md, dev_id))
throw runtime_error("device already exists");
device_tree_detail::device_details details;
details.transaction_id_ = md->sb_.trans_id_;
details.creation_time_ = md->sb_.time_;
details.snapshotted_time_ = details.creation_time_;
md->details_->insert(key, details);
single_mapping_tree::ptr subtree = new_mapping_tree(md);
md->mappings_top_level_->insert(key, subtree->get_root());
md->mappings_->set_root(md->mappings_top_level_->get_root()); // FIXME: ugly
md->commit();
}
metadata::ptr open_or_format_metadata(block_manager::ptr bm, flags const &fs) {
if (fs.op == flags::METADATA_OP_FORMAT)
return format_metadata(bm, fs.data_block_size, fs.nr_data_blocks);
else
return open_metadata(bm);
}
int generate_metadata(flags const &fs) {
block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE);
metadata::ptr md = open_or_format_metadata(bm, fs);
switch (fs.op) {
case flags::METADATA_OP_CREATE_THIN:
create_thin(md, *fs.dev_id);
break;
default:
break;
}
return 0;
@ -137,9 +178,6 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const
<< " {-h|--help}\n"
<< " --data-block-size <block size>\n"
<< " --nr-data-blocks <nr>\n"
<< " --nr-thins <count>\n"
<< " --blocks-per-thin <count>\n"
<< " --run-lengths <count>\n"
<< " {-o|--output} <output device or file>\n"
<< " {-V|--version}" << endl;
}
@ -151,14 +189,15 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
struct flags fs;
const char *shortopts = "hi:o:qV";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "output", required_argument, NULL, 'o'},
{ "data-block-size", required_argument, NULL, 1},
{ "nr-data-blocks", required_argument, NULL, 2},
{ "nr-thins", required_argument, NULL, 3},
{ "blocks-per-thin", required_argument, NULL, 4},
{ "run-lengths", required_argument, NULL, 5},
{ "version", no_argument, NULL, 'V'},
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "format", no_argument, NULL, 1 },
{ "open", no_argument, NULL, 2 },
{ "create-thin", no_argument, NULL, 3 },
{ "data-block-size", required_argument, NULL, 101 },
{ "nr-data-blocks", required_argument, NULL, 102 },
{ "dev-id", required_argument, NULL, 301 },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
@ -173,23 +212,27 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
break;
case 1:
fs.data_block_size = parse_uint64(optarg, "data block size");
fs.op = flags::METADATA_OP_FORMAT;
break;
case 2:
fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
fs.op = flags::METADATA_OP_OPEN;
break;
case 3:
fs.nr_thins = parse_uint64(optarg, "nr thins");
fs.op = flags::METADATA_OP_CREATE_THIN;
break;
case 4:
fs.blocks_per_thin = parse_uint64(optarg, "blocks per thin");
case 101:
fs.data_block_size = parse_uint64(optarg, "data block size");
break;
case 5:
fs.run_lengths = parse_uint64(optarg, "run lengths");
case 102:
fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
break;
case 301:
fs.dev_id = parse_uint64(optarg, "dev id");
break;
case 'V':
@ -202,14 +245,12 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
}
}
if (!fs.output) {
cerr << "No output file provided.\n\n";
if (!fs.check_conformance()) {
usage(cerr);
return 1;
} else
check_output_file_requirements(*fs.output);
}
return create_metadata(fs);
return generate_metadata(fs);
}
//----------------------------------------------------------------