WIP on cache tools

This commit is contained in:
Joe Thornber 2013-09-11 11:40:46 +01:00
parent d3ce6b811b
commit 6615b25e4b
19 changed files with 560 additions and 54 deletions

View File

@ -8,22 +8,22 @@ GEM
builder (3.2.2)
childprocess (0.3.9)
ffi (~> 1.0, >= 1.0.11)
cucumber (1.3.5)
cucumber (1.3.7)
builder (>= 2.1.2)
diff-lcs (>= 1.1.3)
gherkin (~> 2.12.0)
gherkin (~> 2.12.1)
multi_json (~> 1.7.5)
multi_test (>= 0.0.2)
diff-lcs (1.2.4)
ejt_command_line (0.0.2)
ffi (1.9.0)
gherkin (2.12.0)
gherkin (2.12.1)
multi_json (~> 1.3)
multi_json (1.7.7)
multi_json (1.7.9)
multi_test (0.0.2)
rspec-expectations (2.14.0)
rspec-expectations (2.14.2)
diff-lcs (>= 1.1.3, < 2.0)
thinp_xml (0.0.6)
thinp_xml (0.0.8)
ejt_command_line (= 0.0.2)
PLATFORMS

View File

@ -37,10 +37,13 @@ SOURCE=\
base/error_state.cc \
\
caching/superblock.cc \
caching/metadata.cc \
caching/xml_format.cc \
\
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/transaction_manager.cc \
@ -53,7 +56,6 @@ SOURCE=\
persistent-data/space-maps/careful_alloc.cc \
\
thin-provisioning/device_tree.cc \
thin-provisioning/file_utils.cc \
thin-provisioning/human_readable_format.cc \
thin-provisioning/mapping_tree.cc \
thin-provisioning/metadata.cc \
@ -147,6 +149,7 @@ THIN_CHECK_SOURCE=\
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
@ -155,7 +158,6 @@ THIN_CHECK_SOURCE=\
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
thin-provisioning/file_utils.cc \
thin-provisioning/device_tree.cc \
thin-provisioning/mapping_tree.cc \
thin-provisioning/metadata.cc \
@ -166,6 +168,7 @@ THIN_RMAP_SOURCE=\
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
@ -174,7 +177,6 @@ THIN_RMAP_SOURCE=\
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
thin-provisioning/file_utils.cc \
thin-provisioning/device_tree.cc \
thin-provisioning/mapping_tree.cc \
thin-provisioning/metadata.cc \
@ -232,7 +234,10 @@ CACHE_CHECK_SOURCE=\
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
caching/superblock.cc
caching/superblock.cc \
caching/metadata.cc \
caching/xml_format.cc
CACHE_CHECK_OBJECTS=$(subst .cc,.o,$(CACHE_CHECK_SOURCE))
CACHE_DUMP_SOURCE=$(SOURCE)

View File

@ -1,18 +1,41 @@
#include <fstream>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include "version.h"
#include "caching/xml_format.h"
using namespace std;
using namespace caching;
//----------------------------------------------------------------
namespace {
string const STDOUT_PATH("-");
bool want_stdout(string const &output) {
return output == STDOUT_PATH;
}
int dump_(string const &dev, ostream &out) {
emitter::ptr e = create_xml_emitter(out);
return 0;
}
int dump(string const &dev, string const &output) {
if (want_stdout(output)) {
ofstream out(output.c_str());
return dump_(dev, out);
} else
return dump_(dev, cout);
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-o <xml file>}" << endl
<< " {-V|--version}" << endl;
}
}
@ -22,10 +45,12 @@ namespace {
int main(int argc, char **argv)
{
int c;
char const shortopts[] = "hV";
string output("-");
char const shortopts[] = "ho:V";
option const longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "output", required_argument, NULL, 'o'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
@ -36,6 +61,10 @@ int main(int argc, char **argv)
usage(cout, basename(argv[0]));
return 0;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
@ -46,15 +75,13 @@ int main(int argc, char **argv)
}
}
#if 0
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
#endif
return 0;
return dump(argv[optind], output);
}
//----------------------------------------------------------------

View File

@ -1,15 +1,50 @@
#include "version.h"
#include "caching/metadata.h"
#include "caching/xml_format.h"
#include "persistent-data/file_utils.h"
#include <fstream>
#include <getopt.h>
#include <iostream>
#include <libgen.h>
#include <string>
using namespace caching;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
void check_file_exists(string const &file) {
struct stat info;
int r = ::stat(file.c_str(), &info);
if (r)
throw runtime_error("Couldn't stat file");
if (!S_ISREG(info.st_mode))
throw runtime_error("Not a regular file");
}
int restore(string const &xml_file, string const &dev) {
try {
block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY);
metadata::ptr md(new metadata(bm, metadata::CREATE));
emitter::ptr restorer = create_restore_emitter(md);
check_file_exists(xml_file);
ifstream in(xml_file.c_str(), ifstream::in);
parse_xml(in, restorer);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options]" << endl
<< "Options:" << endl
@ -75,7 +110,7 @@ int main(int argc, char **argv)
return 1;
}
return 0;
return restore(input, output);
}
//----------------------------------------------------------------

44
caching/emitter.h Normal file
View File

@ -0,0 +1,44 @@
#ifndef CACHE_EMITTER_H
#define CACHE_EMITTER_H
#include <boost/shared_ptr.hpp>
#include <string>
#include "persistent-data/block.h"
//----------------------------------------------------------------
namespace caching {
namespace pd = persistent_data;
class emitter {
public:
typedef boost::shared_ptr<emitter> ptr;
virtual ~emitter() {}
virtual void begin_superblock(std::string const &uuid,
pd::block_address block_size,
pd::block_address nr_cache_blocks,
std::string const &policy) = 0;
virtual void end_superblock() = 0;
virtual void begin_mappings() = 0;
virtual void end_mappings() = 0;
virtual void mapping(pd::block_address cblock,
pd::block_address oblock,
bool dirty) = 0;
virtual void begin_hints() = 0;
virtual void end_hints() = 0;
virtual void hint(pd::block_address cblock,
std::string const &data) = 0;
};
}
//----------------------------------------------------------------
#endif

17
caching/metadata.cc Normal file
View File

@ -0,0 +1,17 @@
#include "caching/metadata.h"
using namespace caching;
//----------------------------------------------------------------
metadata::metadata(block_manager<>::ptr bm, open_type ot)
{
}
metadata::metadata(block_manager<>::ptr bm, block_address metadata_snap)
{
}
//----------------------------------------------------------------

33
caching/metadata.h Normal file
View File

@ -0,0 +1,33 @@
#ifndef CACHE_METADATA_H
#define CACHE_METADATA_H
#include "persistent-data/block.h"
#include "persistent-data/data-structures/array.h"
#include "persistent-data/endian_utils.h"
#include "persistent-data/space-maps/disk.h"
#include "persistent-data/transaction_manager.h"
#include "caching/superblock.h"
//----------------------------------------------------------------
namespace caching {
class metadata {
public:
enum open_type {
CREATE,
OPEN
};
typedef block_manager<>::read_ref read_ref;
typedef block_manager<>::write_ref write_ref;
typedef boost::shared_ptr<metadata> ptr;
metadata(block_manager<>::ptr bm, open_type ot);
metadata(block_manager<>::ptr bm, block_address metadata_snap);
};
};
//----------------------------------------------------------------
#endif

279
caching/xml_format.cc Normal file
View File

@ -0,0 +1,279 @@
#include "caching/xml_format.h"
#include <boost/lexical_cast.hpp>
#include <expat.h>
using namespace boost;
using namespace caching;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
// base64 encoding?
string encode(string const &data) {
return data;
}
string decode(string const &data) {
return data;
}
//--------------------------------
// Emitter
//--------------------------------
class xml_emitter : public emitter {
public:
xml_emitter(ostream &out)
: out_(out),
indent_(0) {
}
void begin_superblock(std::string const &uuid,
block_address block_size,
block_address nr_cache_blocks,
std::string const &policy) {
indent();
out_ << "<superblock uuid=\"" << uuid << "\""
<< " block_size=\"" << block_size << "\""
<< " nr_cache_blocks=\"" << nr_cache_blocks << "\""
<< " policy=\"" << policy << "\">" << endl;
inc();
}
virtual void end_superblock() {
dec();
indent();
out_ << "</superblock>" << endl;
}
virtual void begin_mappings() {
indent();
out_ << "<mappings>" << endl;
inc();
}
virtual void end_mappings() {
dec();
indent();
out_ << "</mappings>" << endl;
}
virtual void mapping(block_address cblock,
block_address oblock,
bool dirty) {
indent();
out_ << "<mapping"
<< " cache_block=\"" << cblock << "\""
<< " origin_block=\"" << oblock << "\""
<< " dirty=\"" << as_truth(dirty) << "\""
<< "/>" << endl;
}
virtual void begin_hints() {
indent();
out_ << "<hints>" << endl;
inc();
}
virtual void end_hints() {
dec();
indent();
out_ << "</hints>" << endl;
}
virtual void hint(block_address cblock,
std::string const &data) {
out_ << "<hint"
<< " cache_block=\"" << cblock << "\""
<< " data=\"" << encode(data) << "\""
<< ">" << endl;
}
private:
string as_truth(bool v) const {
return v ? "true" : "false";
}
// FIXME: factor out a common class with the thin_provisioning emitter
void indent() {
for (unsigned i = 0; i < indent_ * 2; i++)
out_ << ' ';
}
void inc() {
indent_++;
}
void dec() {
indent_--;
}
ostream &out_;
unsigned indent_;
};
//--------------------------------
// Parser
//--------------------------------
// FIXME: factor out common code with thinp one
typedef std::map<string, string> attributes;
void build_attributes(attributes &a, char const **attr) {
while (*attr) {
char const *key = *attr;
attr++;
if (!*attr) {
ostringstream out;
out << "No value given for xml attribute: " << key;
throw runtime_error(out.str());
}
char const *value = *attr;
a.insert(make_pair(string(key), string(value)));
attr++;
}
}
template <typename T>
T get_attr(attributes const &attr, string const &key) {
attributes::const_iterator it = attr.find(key);
if (it == attr.end()) {
ostringstream out;
out << "could not find attribute: " << key;
throw runtime_error(out.str());
}
return boost::lexical_cast<T>(it->second);
}
template <typename T>
boost::optional<T> get_opt_attr(attributes const &attr, string const &key) {
typedef boost::optional<T> rtype;
attributes::const_iterator it = attr.find(key);
if (it == attr.end())
return rtype();
return rtype(boost::lexical_cast<T>(it->second));
}
void parse_superblock(emitter *e, attributes const &attr) {
e->begin_superblock(get_attr<string>(attr, "uuid"),
get_attr<uint64_t>(attr, "block_size"),
get_attr<uint64_t>(attr, "nr_cache_blocks"),
get_attr<string>(attr, "policy"));
}
bool to_bool(string const &str) {
if (str == "true")
return true;
else if (str == "false")
return false;
throw runtime_error("bad boolean value");
}
void parse_mapping(emitter *e, attributes const &attr) {
e->mapping(get_attr<uint64_t>(attr, "cache_block"),
get_attr<uint64_t>(attr, "origin_block"),
to_bool(get_attr<string>(attr, "dirty")));
}
void parse_hint(emitter *e, attributes const &attr) {
e->hint(get_attr<uint64_t>(attr, "cache_block"),
decode(get_attr<string>(attr, "data")));
}
void start_tag(void *data, char const *el, char const **attr) {
emitter *e = static_cast<emitter *>(data);
attributes a;
build_attributes(a, attr);
if (!strcmp(el, "superblock"))
parse_superblock(e, a);
else if (!strcmp(el, "mappings"))
e->begin_mappings();
else if (!strcmp(el, "mapping"))
parse_mapping(e, a);
else if (!strcmp(el, "hints"))
e->begin_hints();
else if (!strcmp(el, "hint"))
parse_hint(e, a);
else
throw runtime_error("unknown tag type");
}
void end_tag(void *data, const char *el) {
emitter *e = static_cast<emitter *>(data);
if (!strcmp(el, "superblock"))
e->end_superblock();
else if (!strcmp(el, "mappings"))
e->end_mappings();
else if (!strcmp(el, "mapping"))
// do nothing
;
else if (!strcmp(el, "hints"))
e->end_hints();
else if (!strcmp(el, "hint"))
// do nothing
;
else
throw runtime_error("unknown tag close");
}
}
//----------------------------------------------------------------
caching::emitter::ptr
caching::create_xml_emitter(ostream &out)
{
return emitter::ptr(new xml_emitter(out));
}
void
caching::parse_xml(istream &in, emitter::ptr e)
{
XML_Parser parser = XML_ParserCreate(NULL);
if (!parser)
throw runtime_error("couldn't create xml parser");
XML_SetUserData(parser, e.get());
XML_SetElementHandler(parser, start_tag, end_tag);
while (!in.eof()) {
char buffer[4096];
in.read(buffer, sizeof(buffer));
size_t len = in.gcount();
int done = in.eof();
if (!XML_Parse(parser, buffer, len, done)) {
ostringstream out;
out << "Parse error at line "
<< XML_GetCurrentLineNumber(parser)
<< ":\n"
<< XML_ErrorString(XML_GetErrorCode(parser))
<< endl;
throw runtime_error(out.str());
}
}
}
//----------------------------------------------------------------

17
caching/xml_format.h Normal file
View File

@ -0,0 +1,17 @@
#ifndef CACHE_XML_FORMAT_H
#define CACHE_XML_FORMAT_H
#include "emitter.h"
#include <iosfwd>
//----------------------------------------------------------------
namespace caching {
emitter::ptr create_xml_emitter(std::ostream &out);
void parse_xml(std::istream &in, emitter::ptr e);
}
//----------------------------------------------------------------
#endif

View File

@ -28,3 +28,22 @@ Feature: cache_dump
{-h|--help}
{-V|--version}
"""
Scenario: accepts an output file
Given valid cache metadata
When I run cache_dump with -o metadata.xml metadata.bin
Then it should pass
Scenario: missing input file
When I run cache_dump
Then it should fail with:
"""
No input file provided.
"""
Scenario: dump/restore is a noop
Given valid cache metadata
When I cache_dump
And I cache_restore
And I cache_dump
Then cache dumps 1 and 2 should be identical

View File

@ -40,6 +40,10 @@ Feature: thin_restore
No input file provided.
"""
Scenario: input file not found
When I run cache_restore with -i foo.xml -o metadata.bin
Then it should fail
Scenario: missing output file
When I run cache_restore with -i metadata.xml
Then it should fail with:
@ -47,9 +51,8 @@ Feature: thin_restore
No output file provided.
"""
Scenario: dump/restore is a noop
Given valid cache metadata
When I dump cache
And I restore cache
And I dump cache
Then cache dumps 1 and 2 should be identical
Scenario: successfully restores a valid xml file
Given a small xml file
And an empty dev file
When I run cache_restore with -i metadata.xml -o metadata.bin
Then it should pass

View File

@ -64,22 +64,41 @@ When(/^I run cache_restore with (.*?)$/) do |opts|
run_simple("cache_restore #{opts}", false)
end
When(/^I run cache_dump$/) do
run_simple("cache_dump", false)
end
When(/^I run cache_dump with (.*?)$/) do |opts|
run_simple("cache_dump #{opts}", false)
end
Given(/^valid cache metadata$/) do
pending # express the regexp above with the code you wish you had
end
in_current_dir do
system("cache_xml create --nr-cache-blocks uniform[1000-5000] --nr-mappings uniform[500-1000] > #{xml_file}")
end
When(/^I dump cache$/) do
pending # express the regexp above with the code you wish you had
end
When(/^I restore cache$/) do
pending # express the regexp above with the code you wish you had
run_simple("dd if=/dev/zero of=#{dev_file} bs=4k count=1024")
run_simple("cache_restore -i #{xml_file} -o #{dev_file}")
end
Then(/^cache dumps (\d+) and (\d+) should be identical$/) do |arg1, arg2|
pending # express the regexp above with the code you wish you had
run_simple("diff -ub #{dump_files[d1.to_i]} #{dump_files[d2.to_i]}", true)
end
Given(/^a small xml file$/) do
in_current_dir do
system("cache_xml create --nr-cache-blocks 3 --nr-mappings 3 --layout linear --dirty-percent 100 > #{xml_file}")
end
end
Given(/^an empty dev file$/) do
run_simple("dd if=/dev/zero of=#{dev_file} bs=4k count=1024")
end
When(/^I cache_dump$/) do
run_simple("cache_dump #{dev_file} -o #{new_dump_file}", true)
end
When(/^I cache_restore$/) do
run_simple("cache_restore -i #{dump_files[-1]} -o #{dev_file}", true)
end

View File

@ -1,5 +1,5 @@
#include "persistent-data/math_utils.h"
#include "thin-provisioning/file_utils.h"
#include "persistent-data/file_utils.h"
#include <linux/fs.h>
#include <sys/ioctl.h>
@ -12,7 +12,7 @@ using namespace base;
//----------------------------------------------------------------
persistent_data::block_address
thin_provisioning::get_nr_blocks(string const &path)
persistent_data::get_nr_blocks(string const &path)
{
using namespace persistent_data;
@ -47,4 +47,11 @@ thin_provisioning::get_nr_blocks(string const &path)
return nr_blocks;
}
persistent_data::block_manager<>::ptr
persistent_data::open_bm(std::string const &dev_path, block_io<>::mode m)
{
block_address nr_blocks = get_nr_blocks(dev_path);
return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m));
}
//----------------------------------------------------------------

View File

@ -5,8 +5,9 @@
//----------------------------------------------------------------
namespace thin_provisioning {
namespace persistent_data {
persistent_data::block_address get_nr_blocks(string const &path);
block_manager<>::ptr open_bm(std::string const &dev_path, block_io<>::mode m);
}
//----------------------------------------------------------------

View File

@ -17,9 +17,9 @@
// <http://www.gnu.org/licenses/>.
#include "thin-provisioning/device_tree.h"
#include "thin-provisioning/file_utils.h"
#include "thin-provisioning/metadata.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/math_utils.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/space-maps/disk.h"
@ -40,15 +40,6 @@ namespace {
unsigned const METADATA_CACHE_SIZE = 1024;
block_manager<>::ptr open_bm(string const &dev_path, bool writeable) {
block_address nr_blocks = get_nr_blocks(dev_path);
block_io<>::mode m = writeable ?
block_io<>::READ_WRITE :
block_io<>::READ_ONLY;
return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m));
}
transaction_manager::ptr
open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
@ -90,7 +81,7 @@ metadata::metadata(std::string const &dev_path, open_type ot,
{
switch (ot) {
case OPEN:
tm_ = open_tm(open_bm(dev_path, false));
tm_ = open_tm(open_bm(dev_path, block_io<>::READ_ONLY));
sb_ = read_superblock(tm_->get_bm());
if (sb_.version_ != 1)
@ -115,7 +106,7 @@ metadata::metadata(std::string const &dev_path, open_type ot,
break;
case CREATE:
tm_ = open_tm(open_bm(dev_path, true));
tm_ = open_tm(open_bm(dev_path, block_io<>::READ_WRITE));
space_map::ptr core = tm_->get_sm();
metadata_sm_ = create_metadata_sm(tm_, tm_->get_bm()->get_nr_blocks());
copy_space_maps(metadata_sm_, core);
@ -143,7 +134,7 @@ metadata::metadata(std::string const &dev_path, open_type ot,
metadata::metadata(std::string const &dev_path, block_address metadata_snap)
{
tm_ = open_tm(open_bm(dev_path, false));
tm_ = open_tm(open_bm(dev_path, block_io<>::READ_ONLY));
sb_ = read_superblock(tm_->get_bm(), metadata_snap);
// We don't open the metadata sm for a held root

View File

@ -16,7 +16,7 @@
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include "thin-provisioning/file_utils.h"
#include "persistent-data/file_utils.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/metadata_checker.h"

View File

@ -25,8 +25,8 @@
#include "base/error_state.h"
#include "base/nested_output.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/file_utils.h"
#include "thin-provisioning/device_tree.h"
#include "thin-provisioning/file_utils.h"
#include "thin-provisioning/mapping_tree.h"
#include "thin-provisioning/superblock.h"

View File

@ -37,8 +37,8 @@ struct flags {
};
namespace {
int dump(string const &path, ostream *out, string const &format, struct flags &flags,
block_address metadata_snap = 0) {
int dump_(string const &path, ostream &out, string const &format, struct flags &flags,
block_address metadata_snap) {
try {
metadata::ptr md(new metadata(path, metadata_snap));
emitter::ptr e;
@ -56,9 +56,9 @@ namespace {
}
if (format == "xml")
e = create_xml_emitter(*out);
e = create_xml_emitter(out);
else if (format == "human_readable")
e = create_human_readable_emitter(*out);
e = create_human_readable_emitter(out);
else {
cerr << "unknown format '" << format << "'" << endl;
exit(1);
@ -74,6 +74,15 @@ namespace {
return 0;
}
int dump(string const &path, char const *output, string const &format, struct flags &flags,
block_address metadata_snap = 0) {
if (output) {
ofstream out(output);
return dump_(path, out, format, flags, metadata_snap);
} else
return dump_(path, cout, format, flags, metadata_snap);
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
@ -154,5 +163,5 @@ int main(int argc, char **argv)
return 1;
}
return dump(argv[optind], output ? new ofstream(output) : &cout, format, flags, metadata_snap);
return dump(argv[optind], output, format, flags, metadata_snap);
}

View File

@ -9,7 +9,7 @@
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include "persistent-data/run.h"
#include "persistent-data/space-maps/core.h"
#include "thin-provisioning/file_utils.h"
#include "persistent-data/file_utils.h"
#include "thin-provisioning/superblock.h"
#include "thin-provisioning/mapping_tree.h"
#include "thin-provisioning/rmap_visitor.h"