Merge branch 'master' of github.com:jthornber/thin-provisioning-tools

CHANGES

@@ -1,7 +1,34 @@
+v0.6.0
+======
+
+thin_ls
+
+v0.5.6
+======
+
+era_invalidate may be run on live metadata if the --metadata-snap option is given.
+
+v0.5.5
+======
+
+You may now give the --metadata_snap option to thin_delta without specifying where the snap is.
+
+Update man pages to make it clearer that most tools shouldn't be run on live metadata.
+
+Fix some bugs in the metadata reference counting for thin_check.
+
+v0.5.1
+======
+
+Fix bug where the tools would crash if given a very large metadata
+device to restore to.
+
 v0.5
 ====
 
 - thin_delta, thin_trim
+- --clear-needs-check flag for cache_check
+- space map checking for thin check
 
 v0.4
 ====

Makefile.in

@@ -27,9 +27,11 @@ all: $(PROGRAMS)
 SOURCE=\
 	base/application.cc \
 	base/base64.cc \
+	base/disk_units.cc \
 	base/endian_utils.cc \
 	base/error_state.cc \
 	base/error_string.cc \
+	base/grid_layout.cc \
 	base/progress_monitor.cc \
 	base/xml_utils.cc \
 	block-cache/block_cache.cc \
@@ -38,6 +40,7 @@ SOURCE=\
 	caching/cache_metadata_size.cc \
 	caching/cache_repair.cc \
 	caching/cache_restore.cc \
+	caching/commands.cc \
 	caching/hint_array.cc \
 	caching/mapping_array.cc \
 	caching/metadata.cc \
@@ -45,6 +48,7 @@ SOURCE=\
 	caching/restore_emitter.cc \
 	caching/superblock.cc \
 	caching/xml_format.cc \
+	era/commands.cc \
 	era/era_array.cc \
 	era/era_check.cc \
 	era/era_detail.cc \
@@ -70,6 +74,8 @@ SOURCE=\
 	persistent-data/space-maps/recursive.cc \
 	persistent-data/space_map.cc \
 	persistent-data/transaction_manager.cc \
+	persistent-data/validators.cc \
+	thin-provisioning/commands.cc \
 	thin-provisioning/device_tree.cc \
 	thin-provisioning/human_readable_format.cc \
 	thin-provisioning/mapping_tree.cc \
@@ -82,6 +88,7 @@ SOURCE=\
 	thin-provisioning/thin_check.cc \
 	thin-provisioning/thin_delta.cc \
 	thin-provisioning/thin_dump.cc \
+	thin-provisioning/thin_ls.cc \
 	thin-provisioning/thin_metadata_size.cc \
 	thin-provisioning/thin_pool.cc \
 	thin-provisioning/thin_repair.cc \
@@ -92,14 +99,17 @@ SOURCE=\
 
 CC:=@CC@
 CXX:=@CXX@
+STRIP:=@STRIP@
 OBJECTS:=$(subst .cc,.o,$(SOURCE))
 TOP_DIR:=@top_srcdir@
 TOP_BUILDDIR:=@top_builddir@
 CFLAGS+=-g -Wall -O3
+CFLAGS+=@LFS_FLAGS@
 CXXFLAGS+=-g -Wall -fno-strict-aliasing
 CXXFLAGS+=@CXXOPTIMISE_FLAG@
 CXXFLAGS+=@CXXDEBUG_FLAG@
 CXXFLAGS+=@CXX_STRERROR_FLAG@
+CXXFLAGS+=@LFS_FLAGS@
 INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning
 LIBS:=-lstdc++ -laio -lexpat
 INSTALL:=@INSTALL@
@@ -111,7 +121,7 @@ MANPATH:=$(DATADIR)/man
 vpath %.cc $(TOP_DIR)
 
 INSTALL_DIR = $(INSTALL) -m 755 -d
-INSTALL_PROGRAM = $(INSTALL) -m 755 -s
+INSTALL_PROGRAM = $(INSTALL) -m 755
 INSTALL_DATA = $(INSTALL) -p -m 644
 
 ifeq ("@TESTING@", "yes")
@@ -164,6 +174,7 @@ distclean: clean
 install: bin/pdata_tools
 	$(INSTALL_DIR) $(BINDIR)
 	$(INSTALL_PROGRAM) bin/pdata_tools $(BINDIR)
+	$(STRIP) $(BINDIR)/pdata_tools
 	ln -s -f pdata_tools $(BINDIR)/cache_check
 	ln -s -f pdata_tools $(BINDIR)/cache_dump
 	ln -s -f pdata_tools $(BINDIR)/cache_metadata_size
@@ -172,6 +183,7 @@ install: bin/pdata_tools
 	ln -s -f pdata_tools $(BINDIR)/thin_check
 	ln -s -f pdata_tools $(BINDIR)/thin_delta
 	ln -s -f pdata_tools $(BINDIR)/thin_dump
+	ln -s -f pdata_tools $(BINDIR)/thin_ls
 	ln -s -f pdata_tools $(BINDIR)/thin_repair
 	ln -s -f pdata_tools $(BINDIR)/thin_restore
 	ln -s -f pdata_tools $(BINDIR)/thin_rmap
@@ -189,6 +201,7 @@ install: bin/pdata_tools
 	$(INSTALL_DATA) man8/thin_check.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/thin_delta.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/thin_dump.8 $(MANPATH)/man8
+	$(INSTALL_DATA) man8/thin_ls.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/thin_repair.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/thin_restore.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/thin_rmap.8 $(MANPATH)/man8

VERSION

@@ -1 +1 @@
-0.4.1
+0.6.0

base/application.cc

@@ -1,14 +1,47 @@
 #include "base/application.h"
 
+#include <boost/lexical_cast.hpp>
 #include <libgen.h>
 #include <linux/limits.h>
 #include <string.h>
+#include <stdlib.h>
 
 using namespace base;
+using namespace boost;
 using namespace std;
 
 //----------------------------------------------------------------
 
+command::command(string const &name)
+	: name_(name) {
+}
+
+void
+command::die(string const &msg)
+{
+	cerr << msg << endl;
+	usage(cerr);
+	exit(1);
+}
+
+uint64_t
+command::parse_uint64(string const &str, string const &desc)
+{
+	try {
+		// FIXME: check trailing garbage is handled
+		return lexical_cast<uint64_t>(str);
+
+	} catch (...) {
+		ostringstream out;
+		out << "Couldn't parse " << desc << ": '" << str << "'";
+		die(out.str());
+	}
+
+	return 0; // never get here
+}
+
+//----------------------------------------------------------------
+
 int
 application::run(int argc, char **argv)
 {
@@ -26,7 +59,7 @@ application::run(int argc, char **argv)
 		cmd = argv[0];
 	}
 
-	std::list<command const *>::const_iterator it;
+	std::list<command::ptr>::const_iterator it;
 	for (it = cmds_.begin(); it != cmds_.end(); ++it) {
 		if (cmd == (*it)->get_name())
 			return (*it)->run(argc, argv);
@@ -43,7 +76,7 @@ application::usage()
 	std::cerr << "Usage: <command> <args>\n"
 		  << "commands:\n";
 
-	std::list<command const *>::const_iterator it;
+	std::list<command::ptr>::const_iterator it;
 	for (it = cmds_.begin(); it != cmds_.end(); ++it) {
 		std::cerr << "  " << (*it)->get_name() << "\n";
 	}

base/application.h

@@ -1,40 +1,60 @@
 #ifndef BASE_APPLICATION_H
 #define BASE_APPLICATION_H
 
+#include <boost/shared_ptr.hpp>
 #include <iostream>
 #include <list>
 #include <string>
 #include <stdexcept>
+#include <stdint.h>
 
 //----------------------------------------------------------------
 
 namespace base {
 	class command {
 	public:
-		typedef int (*cmd_fn)(int, char **);
+		typedef boost::shared_ptr<command> ptr;
 
-		command(std::string const &name, cmd_fn fn)
-			: name_(name),
-			  fn_(fn) {
-		}
+		command(std::string const &name);
+		virtual ~command() {}
+
+		void die(std::string const &msg);
+		uint64_t parse_uint64(std::string const &str, std::string const &desc);
+
+
+		virtual void usage(std::ostream &out) const = 0;
+		virtual int run(int argc, char **argv) = 0;
 
 		std::string const &get_name() const {
 			return name_;
 		}
 
+	private:
+		std::string name_;
+
+	};
+
+	class command_old : public command {
+	public:
+		typedef int (*cmd_fn)(int, char **);
+
+		command_old(std::string const &name, cmd_fn fn)
+			: command(name),
+			  fn_(fn) {
+		}
+
 		int run(int argc, char **argv) const {
 			return fn_(argc, argv);
 		}
 
 	private:
-		std::string name_;
 		cmd_fn fn_;
 	};
 
 	class application {
 	public:
-		void add_cmd(command const &c) {
-			cmds_.push_back(&c);
+		void add_cmd(command::ptr c) {
+			cmds_.push_back(c);
 		}
 
 		int run(int argc, char **argv);
@@ -43,7 +63,7 @@ namespace base {
 		void usage();
 		std::string get_basename(std::string const &path) const;
 
-		std::list<command const *> cmds_;
+		std::list<command::ptr> cmds_;
 	};
 }
 

base/disk_units.cc

@@ -0,0 +1,95 @@
+#include "base/disk_units.h"
+
+#include <boost/lexical_cast.hpp>
+#include <cmath>
+#include <stdexcept>
+
+using namespace std;
+using namespace boost;
+
+//----------------------------------------------------------------
+
+unsigned long long
+base::disk_unit_multiplier(disk_unit u)
+{
+	switch (u) {
+	case UNIT_BYTE:
+		return 1;
+
+	case UNIT_SECTOR:
+		return 512;
+
+	case UNIT_kB:
+		return 1000;
+
+	case UNIT_MB:
+		return 1000000;
+
+	case UNIT_GB:
+		return 1000000000ull;
+
+	case UNIT_TB:
+		return 1000000000000ull;
+
+	case UNIT_PB:
+		return 1000000000000000ull;
+
+	case UNIT_KiB:
+		return 1024ull;
+
+	case UNIT_MiB:
+		return 1024ull * 1024ull;
+
+	case UNIT_GiB:
+		return 1024ull * 1024ull * 1024ull;
+
+	case UNIT_TiB:
+		return 1024ull * 1024ull * 1024ull * 1024ull;
+
+	case UNIT_PiB:
+		return 1024ull * 1024ull * 1024ull * 1024ull * 1024ull;
+	}
+
+	throw runtime_error("unknown unit type");
+	return 1;
+}
+
+
+namespace {
+	bool small_enough(unsigned long long n) {
+		if (n > 1024ull * 1024ull)
+			return false;
+
+		if (n < 1024ull)
+			return true;
+
+		return (n & 1023) && (n < 8ull * 1024ull);
+	}
+
+	unsigned long long round_ull(unsigned long long n, unsigned long long d) {
+		return round(static_cast<double>(n) / static_cast<double>(d));
+	}
+}
+
+string
+base::format_disk_unit(unsigned long long numerator, disk_unit u)
+{
+	numerator *= disk_unit_multiplier(u);
+	unsigned i;
+	for (i = 0; numerator > 1024ull * 1024ull; i++)
+		numerator /= 1024ull;
+
+	if (numerator > 8 * 1024ull) {
+		numerator = round_ull(numerator, 1024ull);
+		i++;
+	}
+
+	char const *extensions[] = {
+		"", "KiB", "MiB", "GiB", "TiB", "PiB"
+	};
+
+	// FIXME: check subscript of i
+	return lexical_cast<string>(numerator) + extensions[i];
+}
+
+//----------------------------------------------------------------

base/disk_units.h

@@ -0,0 +1,34 @@
+#ifndef BASE_DISK_UNITS_H
+#define BASE_DISK_UNITS_H
+
+#include <string>
+
+//----------------------------------------------------------------
+
+namespace base {
+	enum disk_unit {
+		UNIT_BYTE,
+		UNIT_SECTOR,
+
+		// decimal multipliers
+		UNIT_kB,
+		UNIT_MB,
+		UNIT_GB,
+		UNIT_TB,
+		UNIT_PB,
+
+		// binary multipliers
+		UNIT_KiB,
+		UNIT_MiB,
+		UNIT_GiB,
+		UNIT_TiB,
+		UNIT_PiB
+	};
+
+	unsigned long long disk_unit_multiplier(disk_unit u);
+	std::string format_disk_unit(unsigned long long numerator, disk_unit u);
+}
+
+//----------------------------------------------------------------
+
+#endif

base/grid_layout.cc

@@ -0,0 +1,91 @@
+#include "base/grid_layout.h"
+
+#include <stdexcept>
+
+using namespace base;
+using namespace std;
+
+//----------------------------------------------------------------
+
+grid_layout::grid_layout()
+	: nr_fields_(0)
+{
+	new_row();
+}
+
+void
+grid_layout::render(ostream &out) const
+{
+	vector<unsigned> widths;
+	calc_field_widths(widths);
+
+	grid::const_iterator row;
+	bool newline_needed = false;
+
+	for (row = grid_.begin(); row != grid_.end(); ++row) {
+		row::const_iterator col;
+		unsigned i;
+
+		if (newline_needed) {
+			out << "\n";
+			newline_needed = false;
+		}
+
+		for (col = row->begin(), i = 0; col != row->end(); ++col, ++i) {
+			out << justify(widths[i], *col) << " ";
+			newline_needed = true;
+		}
+	}
+}
+
+void
+grid_layout::new_row()
+{
+	grid_.push_back(row());
+}
+
+grid_layout::row const &
+grid_layout::current_row() const
+{
+	return grid_.back();
+}
+
+grid_layout::row &
+grid_layout::current_row()
+{
+	return grid_.back();
+}
+
+void
+grid_layout::push_field(string const &s)
+{
+	current_row().push_back(s);
+	nr_fields_ = max<unsigned>(nr_fields_, current_row().size());
+}
+
+void
+grid_layout::calc_field_widths(vector<unsigned> &widths) const
+{
+	widths.resize(nr_fields_, 0);
+
+	grid::const_iterator row;
+	for (row = grid_.begin(); row != grid_.end(); ++row) {
+		row::const_iterator col;
+		unsigned i;
+		for (col = row->begin(), i = 0; col != row->end(); ++col, ++i)
+			widths[i] = max<unsigned>(widths[i], col->length());
+	}
+}
+
+string
+grid_layout::justify(unsigned width, string const &txt) const
+{
+	if (txt.length() > width)
+		throw runtime_error("string field too long, internal error");
+
+	string result(width - txt.length(), ' ');
+	result += txt;
+	return result;
+}
+
+//----------------------------------------------------------------

base/grid_layout.h

@@ -0,0 +1,41 @@
+#ifndef BASE_GRID_LAYOUT_H
+#define BASE_GRID_LAYOUT_H
+
+#include <boost/lexical_cast.hpp>
+#include <iosfwd>
+#include <list>
+#include <string>
+#include <vector>
+
+//----------------------------------------------------------------
+
+namespace base {
+	class grid_layout {
+	public:
+		typedef std::list<std::string> row;
+		typedef std::list<row> grid;
+
+		grid_layout();
+		void render(std::ostream &out) const;
+		void new_row();
+
+		template <typename T>
+		void field(T const &t) {
+			push_field(boost::lexical_cast<std::string>(t));
+		}
+
+	private:
+		row &current_row();
+		row const &current_row() const;
+		void push_field(std::string const &s);
+		void calc_field_widths(std::vector<unsigned> &widths) const;
+		std::string justify(unsigned width, std::string const &txt) const;
+
+		grid grid_;
+		unsigned nr_fields_;
+	};
+}
+
+//----------------------------------------------------------------
+
+#endif

bin/thin_ls

@@ -0,0 +1 @@
+pdata_tools

caching/cache_check.cc

@@ -31,7 +31,6 @@ using namespace std;
 //----------------------------------------------------------------
 
 namespace {
-
 	class reporter_base {
 	public:
 		reporter_base(nested_output &o)
@@ -187,7 +186,8 @@ namespace {
 			  check_hints_(true),
 			  check_discards_(true),
 			  ignore_non_fatal_errors_(false),
-			  quiet_(false) {
+			  quiet_(false),
+			  clear_needs_check_on_success_(false) {
 		}
 
 		bool check_mappings_;
@@ -195,6 +195,7 @@ namespace {
 		bool check_discards_;
 		bool ignore_non_fatal_errors_;
 		bool quiet_;
+		bool clear_needs_check_on_success_;
 	};
 
 	struct stat guarded_stat(string const &path) {
@@ -210,7 +211,17 @@ namespace {
 		return info;
 	}
 
-	error_state metadata_check(block_manager<>::ptr bm, flags const &fs) {
+	void clear_needs_check(string const &path) {
+		block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_WRITE);
+
+		superblock sb = read_superblock(bm);
+		sb.flags.clear_flag(superblock_flags::NEEDS_CHECK);
+		write_superblock(bm, sb);
+	}
+
+	error_state metadata_check(string const &path, flags const &fs) {
+		block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY);
+
 		nested_output out(cerr, 2);
 		if (fs.quiet_)
 			out.disable();
@@ -285,8 +296,17 @@ namespace {
 			throw runtime_error(msg.str());
 		}
 
-		block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY);
-		err = metadata_check(bm, fs);
+		err = metadata_check(path, fs);
+
+		bool success = false;
+
+		if (fs.ignore_non_fatal_errors_)
+			success = (err == FATAL) ? false : true;
+		else
+			success = (err == NO_ERROR) ? true : false;
+
+		if (success && fs.clear_needs_check_on_success_)
+			clear_needs_check(path);
 
 		return err == NO_ERROR ? 0 : 1;
 	}
@@ -305,23 +325,32 @@ namespace {
 		return r;
 
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-q|--quiet}" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--super-block-only}" << endl
-		    << "  {--skip-mappings}" << endl
-		    << "  {--skip-hints}" << endl
-		    << "  {--skip-discards}" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int cache_check_main(int argc, char **argv)
+cache_check_cmd::cache_check_cmd()
+	: command("cache_check")
+{
+}
+
+void
+cache_check_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-q|--quiet}" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--clear-needs-check-flag}" << endl
+	    << "  {--super-block-only}" << endl
+	    << "  {--skip-mappings}" << endl
+	    << "  {--skip-hints}" << endl
+	    << "  {--skip-discards}" << endl;
+}
+
+int
+cache_check_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -332,6 +361,7 @@ int cache_check_main(int argc, char **argv)
 		{ "skip-mappings", no_argument, NULL, 2 },
 		{ "skip-hints", no_argument, NULL, 3 },
 		{ "skip-discards", no_argument, NULL, 4 },
+		{ "clear-needs-check-flag", no_argument, NULL, 5 },
 		{ "help", no_argument, NULL, 'h' },
 		{ "version", no_argument, NULL, 'V' },
 		{ NULL, no_argument, NULL, 0 }
@@ -356,8 +386,12 @@ int cache_check_main(int argc, char **argv)
 			fs.check_discards_ = false;
 			break;
 
+		case 5:
+			fs.clear_needs_check_on_success_ = true;
+			break;
+
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'q':
@@ -369,20 +403,18 @@ int cache_check_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return check_with_exception_handling(argv[optind], fs);
 }
 
-base::command caching::cache_check_cmd("cache_check", cache_check_main);
-
 //----------------------------------------------------------------

caching/cache_dump.cc

@@ -54,20 +54,28 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-o <xml file>}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--repair}" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int cache_dump_main(int argc, char **argv)
+cache_dump_cmd::cache_dump_cmd()
+	: command("cache_dump")
+{
+}
+
+void
+cache_dump_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-o <xml file>}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--repair}" << endl;
+}
+
+int
+cache_dump_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -89,7 +97,7 @@ int cache_dump_main(int argc, char **argv)
 			break;
 
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'o':
@@ -101,20 +109,18 @@ int cache_dump_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return dump(argv[optind], output, fs);
 }
 
-base::command caching::cache_dump_cmd("cache_dump", cache_dump_main);
-
 //----------------------------------------------------------------

caching/cache_metadata_size.cc

@@ -9,141 +9,145 @@
 #include <stdint.h>
 #include <stdexcept>
 
+using namespace caching;
 using namespace std;
 
 //----------------------------------------------------------------
 
-namespace {
-	struct flags {
-		flags()
-			: max_hint_width(4) {
+cache_metadata_size_cmd::flags::flags()
+  : max_hint_width(4) {
 
-			// Dance around some spurious compiler warnings
-			device_size = 0;
-			block_size = 0;
-			nr_blocks = 0;
+	// Dance around some spurious compiler warnings
+	device_size = 0;
+	block_size = 0;
+	nr_blocks = 0;
 
-			device_size.reset();
-			block_size.reset();
-			nr_blocks.reset();
-		}
-
-		boost::optional<uint64_t> device_size;
-		boost::optional<uint32_t> block_size;
-		boost::optional<uint64_t> nr_blocks;
-		uint32_t max_hint_width;
-	};
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options]" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--block-size <sectors>}" << endl
-		    << "  {--device-size <sectors>}" << endl
-		    << "  {--nr-blocks <natural>}" << endl << endl
-		    << "These all relate to the size of the fast device (eg, SSD), rather" << endl
-		    << "than the whole cached device." << endl;
-	}
-
-	enum parse_result {
-		FINISH,
-		CONTINUE
-	};
-
-	parse_result parse_command_line(string const &prog_name, int argc, char **argv, flags &fs) {
-
-		int c;
-		char const short_opts[] = "hV";
-		option const long_opts[] = {
-			{ "block-size", required_argument, NULL, 0 },
-			{ "device-size", required_argument, NULL, 1 },
-			{ "nr-blocks", required_argument, NULL, 2 },
-			{ "max-hint-width", required_argument, NULL, 3 },
-			{ "help", no_argument, NULL, 'h' },
-			{ "version", no_argument, NULL, 'V' },
-			{ NULL, no_argument, NULL, 0 }
-		};
-
-		while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
-			switch (c) {
-			case 0:
-				fs.block_size = boost::lexical_cast<uint32_t>(optarg);
-				break;
-
-			case 1:
-				fs.device_size = boost::lexical_cast<uint64_t>(optarg);
-				break;
-
-			case 2:
-				fs.nr_blocks = boost::lexical_cast<uint64_t>(optarg);
-				break;
-
-			case 3:
-				fs.max_hint_width = boost::lexical_cast<uint32_t>(optarg);
-				break;
-
-			case 'h':
-				usage(cout, prog_name);
-				return FINISH;
-				break;
-
-			case 'V':
-				cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
-				return FINISH;
-				break;
-
-			default:
-				usage(cerr, prog_name);
-				throw runtime_error("Invalid command line");
-				break;
-			}
-		}
-
-		return CONTINUE;
-	}
-
-  	uint64_t get_nr_blocks(flags &fs) {
-		if (fs.device_size) {
-			if (!fs.block_size)
-				throw runtime_error("If you specify --device-size you must also give --block-size.");
-
-			uint64_t nr_blocks = *fs.device_size / *fs.block_size;
-			if (fs.nr_blocks) {
-				if (nr_blocks != *fs.nr_blocks)
-					throw runtime_error(
-						"Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size.");
-			}
-
-			return nr_blocks;
-		}
-
-		if (fs.block_size && !fs.device_size)
-			throw runtime_error("If you specify --block-size you must also give --device-size.");
-
-		if (fs.nr_blocks)
-			return *fs.nr_blocks;
-
-		throw runtime_error("Please specify either --device-size and --block-size, or --nr-blocks.");
-	}
-
-	uint64_t meg(uint64_t n) {
-		return n * 2048;
-	}
-
-  uint64_t calc_size(uint64_t nr_blocks, uint32_t max_hint_width) {
-		uint64_t const SECTOR_SIZE = 512;
-		uint64_t const TRANSACTION_OVERHEAD = meg(4);
-		uint64_t const BYTES_PER_BLOCK = 16;
-		uint64_t const HINT_OVERHEAD_PER_BLOCK = 8;
-
-		uint64_t mapping_size = (nr_blocks * BYTES_PER_BLOCK) / SECTOR_SIZE;
-		uint64_t hint_size = (nr_blocks * (max_hint_width + HINT_OVERHEAD_PER_BLOCK)) / SECTOR_SIZE;
-		return TRANSACTION_OVERHEAD + mapping_size + hint_size;
-	}
+	device_size.reset();
+	block_size.reset();
+	nr_blocks.reset();
 }
 
-int cache_metadata_size_main(int argc, char **argv)
+cache_metadata_size_cmd::parse_result
+cache_metadata_size_cmd::parse_command_line(string const &prog_name, int argc, char **argv, flags &fs)
+{
+	int c;
+	char const short_opts[] = "hV";
+	option const long_opts[] = {
+		{ "block-size", required_argument, NULL, 0 },
+		{ "device-size", required_argument, NULL, 1 },
+		{ "nr-blocks", required_argument, NULL, 2 },
+		{ "max-hint-width", required_argument, NULL, 3 },
+		{ "help", no_argument, NULL, 'h' },
+		{ "version", no_argument, NULL, 'V' },
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
+		switch (c) {
+		case 0:
+			fs.block_size = boost::lexical_cast<uint32_t>(optarg);
+			break;
+
+		case 1:
+			fs.device_size = boost::lexical_cast<uint64_t>(optarg);
+			break;
+
+		case 2:
+			fs.nr_blocks = boost::lexical_cast<uint64_t>(optarg);
+			break;
+
+		case 3:
+			fs.max_hint_width = boost::lexical_cast<uint32_t>(optarg);
+			break;
+
+		case 'h':
+			usage(cout);
+			return FINISH;
+			break;
+
+		case 'V':
+			cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
+			return FINISH;
+			break;
+
+		default:
+			usage(cerr);
+			throw runtime_error("Invalid command line");
+			break;
+		}
+	}
+
+	return CONTINUE;
+}
+
+uint64_t
+cache_metadata_size_cmd::get_nr_blocks(flags &fs)
+{
+	if (fs.device_size) {
+		if (!fs.block_size)
+			throw runtime_error("If you specify --device-size you must also give --block-size.");
+
+		uint64_t nr_blocks = *fs.device_size / *fs.block_size;
+		if (fs.nr_blocks) {
+			if (nr_blocks != *fs.nr_blocks)
+				throw runtime_error(
+					"Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size.");
+		}
+
+		return nr_blocks;
+	}
+
+	if (fs.block_size && !fs.device_size)
+		throw runtime_error("If you specify --block-size you must also give --device-size.");
+
+	if (fs.nr_blocks)
+		return *fs.nr_blocks;
+
+	throw runtime_error("Please specify either --device-size and --block-size, or --nr-blocks.");
+}
+
+uint64_t
+cache_metadata_size_cmd::meg(uint64_t n)
+{
+	return n * 2048;
+}
+
+uint64_t
+cache_metadata_size_cmd::calc_size(uint64_t nr_blocks, uint32_t max_hint_width)
+{
+	uint64_t const SECTOR_SIZE = 512;
+	uint64_t const TRANSACTION_OVERHEAD = meg(4);
+	uint64_t const BYTES_PER_BLOCK = 16;
+	uint64_t const HINT_OVERHEAD_PER_BLOCK = 8;
+
+	uint64_t mapping_size = (nr_blocks * BYTES_PER_BLOCK) / SECTOR_SIZE;
+	uint64_t hint_size = (nr_blocks * (max_hint_width + HINT_OVERHEAD_PER_BLOCK)) / SECTOR_SIZE;
+	return TRANSACTION_OVERHEAD + mapping_size + hint_size;
+}
+
+//----------------------------------------------------------------
+
+cache_metadata_size_cmd::cache_metadata_size_cmd()
+	: command("cache_metadata_size")
+{
+}
+
+void
+cache_metadata_size_cmd::usage(ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options]" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--block-size <sectors>}" << endl
+	    << "  {--device-size <sectors>}" << endl
+	    << "  {--nr-blocks <natural>}" << endl << endl
+	    << "These all relate to the size of the fast device (eg, SSD), rather" << endl
+	    << "than the whole cached device." << endl;
+}
+
+int
+cache_metadata_size_cmd::run(int argc, char **argv)
 {
 	flags fs;
 
@@ -167,6 +171,4 @@ int cache_metadata_size_main(int argc, char **argv)
 	return 0;
 }
 
-base::command caching::cache_metadata_size_cmd("cache_metadata_size", cache_metadata_size_main);
-
 //----------------------------------------------------------------

caching/cache_repair.cc

@@ -40,20 +40,28 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-i|--input} <input metadata (binary format)>" << endl
-		    << "  {-o|--output} <output metadata (binary format)>" << endl
-		    << "  {-V|--version}" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int cache_repair_main(int argc, char **argv)
+cache_repair_cmd::cache_repair_cmd()
+	: command("cache_repair")
+{
+}
+
+void
+cache_repair_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-i|--input} <input metadata (binary format)>" << endl
+	    << "  {-o|--output} <output metadata (binary format)>" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+cache_repair_cmd::run(int argc, char **argv)
 {
 	int c;
 	boost::optional<string> input_path, output_path;
@@ -70,7 +78,7 @@ int cache_repair_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
 		switch(c) {
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'i':
@@ -86,26 +94,24 @@ int cache_repair_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (!input_path) {
 		cerr << "no input file provided" << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	if (!output_path) {
 		cerr << "no output file provided" << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return repair(*input_path, *output_path);
 }
 
-base::command caching::cache_repair_cmd("cache_repair", cache_repair_main);
-
 //----------------------------------------------------------------

caching/cache_restore.cc

@@ -80,27 +80,35 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options]" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-i|--input} <input xml file>" << endl
-		    << "  {-o|--output} <output device or file>" << endl
-		    << "  {-q|--quiet}" << endl
-		    << "  {-V|--version}" << endl
-		    << endl
-		    << "  {--debug-override-metadata-version} <integer>" << endl
-		    << "  {--omit-clean-shutdown}" << endl;
-
-	}
 }
 
-int cache_restore_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+cache_restore_cmd::cache_restore_cmd()
+	: command("cache_restore")
+{
+}
+
+void
+cache_restore_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options]" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-i|--input} <input xml file>" << endl
+	    << "  {-o|--output} <output device or file>" << endl
+	    << "  {-q|--quiet}" << endl
+	    << "  {-V|--version}" << endl
+	    << endl
+	    << "  {--debug-override-metadata-version} <integer>" << endl
+	    << "  {--omit-clean-shutdown}" << endl;
+}
+
+int
+cache_restore_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
-	char const *prog_name = basename(argv[0]);
 	char const *short_opts = "hi:o:qV";
 	option const long_opts[] = {
 		{ "debug-override-metadata-version", required_argument, NULL, 0 },
@@ -125,7 +133,7 @@ int cache_restore_main(int argc, char **argv)
 			break;
 
 		case 'h':
-			usage(cout, prog_name);
+			usage(cout);
 			return 0;
 
 		case 'i':
@@ -145,31 +153,29 @@ int cache_restore_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, prog_name);
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc != optind) {
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
         if (!fs.input) {
 		cerr << "No input file provided." << endl << endl;
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
 	if (!fs.output) {
 		cerr << "No output file provided." << endl << endl;
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
 	return restore(fs);
 }
 
-base::command caching::cache_restore_cmd("cache_restore", cache_restore_main);
-
 //----------------------------------------------------------------

caching/commands.cc

@@ -0,0 +1,18 @@
+#include "caching/commands.h"
+
+using namespace base;
+using namespace caching;
+
+//----------------------------------------------------------------
+
+void
+caching::register_cache_commands(application &app)
+{
+	app.add_cmd(command::ptr(new cache_check_cmd));
+	app.add_cmd(command::ptr(new cache_dump_cmd));
+	app.add_cmd(command::ptr(new cache_metadata_size_cmd));
+	app.add_cmd(command::ptr(new cache_restore_cmd));
+	app.add_cmd(command::ptr(new cache_repair_cmd));
+}
+
+//----------------------------------------------------------------

caching/commands.h

@@ -2,15 +2,68 @@
 #define CACHING_COMMANDS_H
 
 #include "base/application.h"
+#include "boost/optional.hpp"
 
 //----------------------------------------------------------------
 
 namespace caching {
-	extern base::command cache_check_cmd;
-	extern base::command cache_dump_cmd;
-	extern base::command cache_metadata_size_cmd;
-	extern base::command cache_restore_cmd;
-	extern base::command cache_repair_cmd;
+	class cache_check_cmd : public base::command {
+	public:
+		cache_check_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class cache_dump_cmd : public base::command {
+	public:
+		cache_dump_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class cache_metadata_size_cmd : public base::command {
+	public:
+		cache_metadata_size_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+
+
+	private:
+		struct flags {
+			flags();
+
+			boost::optional<uint64_t> device_size;
+			boost::optional<uint32_t> block_size;
+			boost::optional<uint64_t> nr_blocks;
+			uint32_t max_hint_width;
+		};
+
+		enum parse_result {
+			FINISH,
+			CONTINUE
+		};
+
+		parse_result parse_command_line(std::string const &prog_name, int argc, char **argv, flags &fs);
+		uint64_t get_nr_blocks(flags &fs);
+		uint64_t meg(uint64_t n);
+		uint64_t calc_size(uint64_t nr_blocks, uint32_t max_hint_width);
+	};
+
+	class cache_repair_cmd : public base::command {
+	public:
+		cache_repair_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class cache_restore_cmd : public base::command {
+	public:
+		cache_restore_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	void register_cache_commands(base::application &app);
 }
 
 //----------------------------------------------------------------

caching/superblock.cc

@@ -74,6 +74,11 @@ superblock_flags::superblock_flags(uint32_t bits)
 		bits &= ~(1 << CLEAN_SHUTDOWN_BIT);
 	}
 
+	if (bits & (1u << NEEDS_CHECK_BIT)) {
+		flags_.insert(NEEDS_CHECK);
+		bits &= ~(1u << NEEDS_CHECK_BIT);
+	}
+
 	unhandled_flags_ = bits;
 }
 
@@ -103,6 +108,9 @@ superblock_flags::encode() const
 	if (get_flag(CLEAN_SHUTDOWN))
 		r = r | (1 << CLEAN_SHUTDOWN_BIT);
 
+	if (get_flag(NEEDS_CHECK))
+		r = r | (1u << NEEDS_CHECK_BIT);
+
 	return r;
 }
 

caching/superblock.h

@@ -19,11 +19,13 @@ namespace caching {
 	class superblock_flags {
 	public:
 		enum flag {
-			CLEAN_SHUTDOWN
+			CLEAN_SHUTDOWN,
+			NEEDS_CHECK
 		};
 
 		enum flag_bits {
-			CLEAN_SHUTDOWN_BIT = 0
+			CLEAN_SHUTDOWN_BIT = 0,
+			NEEDS_CHECK_BIT = 1,
 		};
 
 		superblock_flags();

configure.ac

@@ -41,6 +41,19 @@ AC_PROG_LN_S
 AC_PROG_MAKE_SET
 AC_PROG_MKDIR_P
 AC_PROG_INSTALL
+AC_DEFUN([AC_PROG_STRIP], [AC_CHECK_TOOL(STRIP, strip, :)])
+AC_PROG_STRIP
+AC_ARG_VAR(STRIP, [Command for discarding symbols from object files])
+
+################################################################
+dnl -- Check for large file support
+AC_SYS_LARGEFILE
+if test x$ac_cv_sys_file_offset_bits = x64; then
+    LFS_FLAGS+="-D_FILE_OFFSET_BITS=64"
+fi
+if test x$ac_cv_sys_large_files = x1; then
+    LFS_FLAGS+="-D_LARGE_FILES"
+fi
 
 ################################################################
 dnl -- Checks for functions.
@@ -144,6 +157,7 @@ VERSION_PATCHLEVEL=`echo "$VER" | $AWK -F '[[(.]]' '{print $3}'`
 AC_SUBST(CXXDEBUG_FLAG)
 AC_SUBST(CXXOPTIMISE_FLAG)
 AC_SUBST(CXX_STRERROR_FLAG)
+AC_SUBST(LFS_FLAGS)
 AC_SUBST(INSTALL)
 AC_SUBST(prefix)
 AC_SUBST(RELEASE_DATE)

era/commands.cc

@@ -0,0 +1,16 @@
+#include "era/commands.h"
+
+using namespace base;
+
+//----------------------------------------------------------------
+
+void
+era::register_era_commands(base::application &app)
+{
+	app.add_cmd(command::ptr(new era_check_cmd()));
+	app.add_cmd(command::ptr(new era_dump_cmd()));
+	app.add_cmd(command::ptr(new era_invalidate_cmd()));
+	app.add_cmd(command::ptr(new era_restore_cmd()));
+}
+
+//----------------------------------------------------------------

era/commands.h

@@ -6,10 +6,35 @@
 //----------------------------------------------------------------
 
 namespace era {
-	extern base::command era_check_cmd;
-	extern base::command era_dump_cmd;
-	extern base::command era_invalidate_cmd;
-	extern base::command era_restore_cmd;
+	class era_check_cmd : public base::command {
+	public:
+		era_check_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class era_dump_cmd : public base::command {
+	public:
+		era_dump_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class era_invalidate_cmd : public base::command {
+	public:
+		era_invalidate_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class era_restore_cmd : public base::command {
+	public:
+		era_restore_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	void register_era_commands(base::application &app);
 }
 
 //----------------------------------------------------------------

era/era_check.cc

@@ -262,20 +262,28 @@ namespace {
 		return r;
 
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-q|--quiet}" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--super-block-only}" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int era_check_main(int argc, char **argv)
+era_check_cmd::era_check_cmd()
+	: command("era_check")
+{
+}
+
+void
+era_check_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-q|--quiet}" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--super-block-only}" << endl;
+}
+
+int
+era_check_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -295,7 +303,7 @@ int era_check_main(int argc, char **argv)
 			break;
 
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'q':
@@ -307,20 +315,18 @@ int era_check_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return check_with_exception_handling(argv[optind], fs);
 }
 
-base::command era::era_check_cmd("era_check", era_check_main);
-
 //----------------------------------------------------------------

era/era_dump.cc

@@ -57,21 +57,29 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-o <xml file>}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--repair}" << endl
-		    << "  {--logical}" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int era_dump_main(int argc, char **argv)
+era_dump_cmd::era_dump_cmd()
+	: command("era_dump")
+{
+}
+
+void
+era_dump_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-o <xml file>}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--repair}" << endl
+	    << "  {--logical}" << endl;
+}
+
+int
+era_dump_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -98,7 +106,7 @@ int era_dump_main(int argc, char **argv)
 			break;
 
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'o':
@@ -110,20 +118,18 @@ int era_dump_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return dump(argv[optind], output, fs);
 }
 
-base::command era::era_dump_cmd("era_dump", era_dump_main);
-
 //----------------------------------------------------------------

era/era_invalidate.cc

@@ -152,7 +152,7 @@ namespace {
 	int invalidate(string const &dev, string const &output, flags const &fs) {
 		try {
 			set<uint32_t> blocks;
-			block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY);
+			block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY, !fs.metadata_snapshot_);
 
 			if (fs.metadata_snapshot_) {
 				superblock sb = read_superblock(bm);
@@ -182,20 +182,28 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] --written-since <era> {device|file}\n"
-		    << "Options:\n"
-		    << "  {-h|--help}\n"
-		    << "  {-o <xml file>}\n"
-		    << "  {--metadata-snapshot}\n"
-		    << "  {-V|--version}" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int era_invalidate_main(int argc, char **argv)
+era_invalidate_cmd::era_invalidate_cmd()
+	: command("era_invalidate")
+{
+}
+
+void
+era_invalidate_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] --written-since <era> {device|file}\n"
+	    << "Options:\n"
+	    << "  {-h|--help}\n"
+	    << "  {-o <xml file>}\n"
+	    << "  {--metadata-snapshot}\n"
+	    << "  {-V|--version}" << endl;
+}
+
+int
+era_invalidate_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -222,7 +230,7 @@ int era_invalidate_main(int argc, char **argv)
 			break;
 
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'o':
@@ -234,26 +242,24 @@ int era_invalidate_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	if (!fs.era_threshold_) {
 		cerr << "Please specify --written-since" << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return invalidate(argv[optind], output, fs);
 }
 
-base::command era::era_invalidate_cmd("era_invalidate", era_invalidate_main);
-
 //----------------------------------------------------------------

era/era_restore.cc

@@ -46,23 +46,32 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options]" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-i|--input} <input xml file>" << endl
-		    << "  {-o|--output} <output device or file>" << endl
-		    << "  {-q|--quiet}" << endl
-		    << "  {-V|--version}" << endl;
-	}
 }
 
-int era_restore_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+era_restore_cmd::era_restore_cmd()
+	: command("era_restore")
+{
+}
+
+void
+era_restore_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options]" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-i|--input} <input xml file>" << endl
+	    << "  {-o|--output} <output device or file>" << endl
+	    << "  {-q|--quiet}" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+era_restore_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
-	char const *prog_name = basename(argv[0]);
 	char const *short_opts = "hi:o:qV";
 	option const long_opts[] = {
 		{ "help", no_argument, NULL, 'h'},
@@ -76,7 +85,7 @@ int era_restore_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
 		switch(c) {
 		case 'h':
-			usage(cout, prog_name);
+			usage(cout);
 			return 0;
 
 		case 'i':
@@ -96,31 +105,29 @@ int era_restore_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, prog_name);
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc != optind) {
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
         if (!fs.input) {
 		cerr << "No input file provided." << endl << endl;
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
 	if (!fs.output) {
 		cerr << "No output file provided." << endl << endl;
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
 	return restore(fs, fs.quiet);
 }
 
-base::command era::era_restore_cmd("era_restore", era_restore_main);
-
 //----------------------------------------------------------------

features/cache_check.feature

@@ -93,3 +93,8 @@ Feature: cache_check
     And I run cache_restore with -i metadata.xml -o input --debug-override-metadata-version 12345
     When I run `cache_check input`
     Then it should fail
+
+  Scenario: Accepts --clear-needs-check-flag
+    Given valid cache metadata
+    When I run `cache_check --clear-needs-check-flag metadata.bin`
+    Then it should pass

features/step_definitions/cache_steps.rb

@@ -40,6 +40,7 @@ Options:
   {-q|--quiet}
   {-h|--help}
   {-V|--version}
+  {--clear-needs-check-flag}
   {--super-block-only}
   {--skip-mappings}
   {--skip-hints}

features/thin_delta.feature

@@ -12,8 +12,11 @@ Feature: thin_delta
     Then it should pass with:
 
     """
-    Usage: thin_delta [options] --snap1 <snap> --snap2 <snap> <device or file>
+    Usage: thin_delta [options] <device or file>
     Options:
+      {--thin1, --snap1}
+      {--thin2, --snap2}
+      {-m, --metadata-snap} [block#]
       {--verbose}
       {-h|--help}
       {-V|--version}
@@ -23,8 +26,11 @@ Feature: thin_delta
     When I run `thin_delta -h`
     Then it should pass with:
     """
-    Usage: thin_delta [options] --snap1 <snap> --snap2 <snap> <device or file>
+    Usage: thin_delta [options] <device or file>
     Options:
+      {--thin1, --snap1}
+      {--thin2, --snap2}
+      {-m, --metadata-snap} [block#]
       {--verbose}
       {-h|--help}
       {-V|--version}

main.cc

@@ -14,27 +14,9 @@ int main(int argc, char **argv)
 
 	application app;
 
-	app.add_cmd(caching::cache_check_cmd);
-	app.add_cmd(caching::cache_dump_cmd);
-	app.add_cmd(caching::cache_metadata_size_cmd);
-	app.add_cmd(caching::cache_restore_cmd);
-	app.add_cmd(caching::cache_repair_cmd);
-
-	app.add_cmd(era::era_check_cmd);
-	app.add_cmd(era::era_dump_cmd);
-	app.add_cmd(era::era_invalidate_cmd);
-	app.add_cmd(era::era_restore_cmd);
-
-	app.add_cmd(thin_provisioning::thin_check_cmd);
-	app.add_cmd(thin_provisioning::thin_delta_cmd);
-	app.add_cmd(thin_provisioning::thin_dump_cmd);
-	app.add_cmd(thin_provisioning::thin_metadata_size_cmd);
-	app.add_cmd(thin_provisioning::thin_restore_cmd);
-	app.add_cmd(thin_provisioning::thin_repair_cmd);
-	app.add_cmd(thin_provisioning::thin_rmap_cmd);
-
-	// FIXME: convert thin_metadata_size to c++
-	//app.add_cmd(thin_provisioning::thin_metadata_size_cmd);
+	caching::register_cache_commands(app);
+	era::register_era_commands(app);
+	thin_provisioning::register_thin_commands(app);
 
 	return app.run(argc, argv);
 }

man8/cache_check.8

@@ -15,6 +15,8 @@ the device-mapper cache target on a
 or
 .I file.
 
+This tool cannot be run on live metadata.
+
 .SH OPTIONS
 .IP "\fB\-q, \-\-quiet\fP"
 Suppress output messages, return only exit code.

man8/cache_dump.8

@@ -23,6 +23,8 @@ in order to put it back onto a metadata
 (to process by the device-mapper target) or
 .I file.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\-r, \-\-repair\fP".
 Repair the metadata whilst dumping it.
 

man8/cache_repair.8

@@ -26,6 +26,8 @@ If written to a metadata
 , the metadata can be processed
 by the device-mapper target.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\-i, \-\-input\fP \fI{device|file}\fP"
 Input file or device with binary metadata.
 

man8/cache_restore.8

@@ -24,6 +24,8 @@ If restored to a metadata
 .I device
 , the metadata can be processed by the device-mapper target.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\-i, \-\-input\fP \fI{device|file}\fP"
 Input file or device with metadata.
 

man8/era_check.8

@@ -15,6 +15,8 @@ the device-mapper era target on a
 or
 .I file.
 
+This tool cannot be run on live metadata.
+
 .SH OPTIONS
 .IP "\fB\-q, \-\-quiet\fP"
 Suppress output messages, return only exit code.

man8/era_dump.8

@@ -23,6 +23,8 @@ in order to put it back onto a metadata
 (to process by the device-mapper target) or
 .I file.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\-r, \-\-repair\fP".
 Repair the metadata whilst dumping it.
 

man8/era_invalidate.8

@@ -11,6 +11,8 @@ era_invalidate \- Provide a list of blocks that have changed since a particular
 .B era_invalidate
 Examines era metadata and lists blocks that may have changed since a given era.
 
+This tool cannot be run on live metadata unless the \fB\-\-metadata\-snap\fP option is used.
+
 .SH OPTIONS
 .IP "\fB\-h, \-\-help\fP"
 Print help and exit.

man8/thin_check.8

@@ -15,6 +15,8 @@ the device-mapper thin provisioning target on a
 or
 .I file.
 
+This tool cannot be run on live metadata.
+
 .SH OPTIONS
 .IP "\fB\-q, \-\-quiet\fP"
 Suppress output messages, return only exit code.

man8/thin_delta.8

@@ -10,7 +10,8 @@ thin_delta \- Print the differences in the mappings between two thin devices.
 .SH DESCRIPTION
 .B thin_delta
 allows you to compare the mappings in two thin volumes (snapshots allow common blocks between thin volumes).
-.
+
+This tool cannot be run on live metadata unless the \fB\-\-metadata\-snap\fP option is used.
 
 .SH OPTIONS
 .IP "\fB\-\-thin1, \-\-snap1\fP"

man8/thin_dump.8

@@ -24,13 +24,15 @@ in order to put it back onto a metadata
 (to process by the device-mapper target) or
 .I file.
 
+This tool cannot be run on live metadata unless the \fB\-\-metadata\-snap\fP option is used.
+
 .IP "\fB\-f, \-\-format\fP \fI{xml|human_readable}\fP".
 Print output in XML or human readable format.
 
 .IP "\fB\-r, \-\-repair\fP".
 Repair the metadata whilst dumping it.
 
-.IP "\fB\-m, \-\-metadata_snap\fP [block#]".
+.IP "\fB\-m, \-\-metadata\-snap\fP [block#]".
 Dump metadata snapshot created by device-mapper thin provisioning target.
 If block is not provided, access the default metadata snapshot created by
 the thin provisioning device-mapper target, else try the one at block#.

man8/thin_ls.8

@@ -0,0 +1,51 @@
+.TH THIN_LS 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*-
+.SH NAME
+thin_ls \- List thin volumes within a pool.
+
+.SH SYNOPSIS
+.B thin_ls
+.RB [ options ]
+.I {metadata device}
+
+.SH DESCRIPTION
+.B thin_ls
+Displays infomation about thin volumes in a pool.  Pass the metadata device on the command line, not the
+pool device.
+
+This tool cannot be run on live metadata unless the \fB\-\-metadata\-snap\fP option is used.
+
+.SH OPTIONS
+
+.IP "\fB\-o, \-\-format\fP"
+Give a comma separated list of the fields to be output.  Valid fields are:
+DEV, MAPPED_BLOCKS, EXCLUSIVE_BLOCKS, SHARED_BLOCKS, MAPPED_SECTORS,
+EXCLUSIVE_SECTORS, SHARED_SECTORS, MAPPED, EXCLUSIVE, SHARED, TRANSACTION
+CREATE_TIME, SNAP_TIME
+
+.IP "\fB\-\-no\-headers\fP"
+Don't output headers.
+
+.IP "\fB\-m, \-\-metadata\-snap\fP"
+
+If you want to get information out of a live pool then you will need
+to take a metadata snapshot and use this switch.
+
+.IP "\fB\-\-verbose"
+Provide extra information on the mappings.
+
+.IP "\fB\-h, \-\-help\fP"
+Print help and exit.
+
+.IP "\fB\-V, \-\-version\fP"
+Output version information and exit.
+
+.SH SEE ALSO
+.B thin_dump(8)
+.B thin_repair(8)
+.B thin_restore(8)
+.B thin_rmap(8)
+.B thin_trim(8)
+.B thin_metadata_size(8)
+
+.SH AUTHOR
+Joe Thornber <ejt@redhat.com>

man8/thin_repair.8

@@ -26,6 +26,8 @@ If written to a metadata
 , the metadata can be processed
 by the device-mapper target.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\-i, \-\-input\fP \fI{device|file}\fP"
 Input file or device with binary metadata.
 

man8/thin_restore.8

@@ -24,6 +24,8 @@ If restored to a metadata
 .I device
 , the metadata can be processed by the device-mapper target.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\-q, \-\-quiet\fP"
 Suppress output messages, return only exit code.
 

man8/thin_rmap.8

@@ -16,6 +16,8 @@ or
 between a region of thin provisioned pool blocks
 and the associated thin provisioned devices.
 
+This tool cannot be run on live metadata.
+
 .IP "\fB\\-\-region\fP \fI<block range>\fP".
 output reverse map
 
@@ -29,7 +31,7 @@ Output version information and exit.
 output reverse map for pool blocks 5..45 (denotes blocks 5 to 44 inclusive, but not block 45)
 
 .sp
-.B thin_rmap --region 5..45 /dev/vg/pool
+.B thin_rmap --region 5..45 /dev/pool-metadata
 
 .SH DIAGNOSTICS
 .B thin_rmap

man8/thin_trim.8

@@ -9,9 +9,9 @@ thin_trim \- Issue discard requests for free pool space (offline tool).
 
 .SH DESCRIPTION
 .B thin_trim
-sends discard requests to the pool device for unprovisioned areas.  It is an offline tool, 
-.B do not run it while the pool is active
-.
+sends discard requests to the pool device for unprovisioned areas.
+
+This tool cannot be run on live metadata.
 
 .SH OPTIONS
 .IP "\fB\-\-pool-inactive\fP"

persistent-data/block.h

@@ -52,7 +52,8 @@ namespace persistent_data {
 		block_manager(std::string const &path,
 			      block_address nr_blocks,
 			      unsigned max_concurrent_locks,
-			      mode m);
+			      mode m,
+			      bool excl = true);
 
 		class read_ref {
 		public:
@@ -134,7 +135,8 @@ namespace persistent_data {
 		bool is_locked(block_address b) const;
 
 	private:
-		int open_or_create_block_file(std::string const &path, off_t file_size, mode m);
+		int open_or_create_block_file(std::string const &path, off_t file_size,
+					      mode m, bool excl);
 		void check(block_address b) const;
 
 		int fd_;

persistent-data/block.tcc

@@ -46,14 +46,23 @@ namespace {
 	// to exception.h
 	void syscall_failed(char const *call) {
 		ostringstream out;
-		out << "syscall '" << call << "' failed: " << base::error_string(errno);;
+		out << "syscall '" << call << "' failed: " << base::error_string(errno);
+		throw runtime_error(out.str());
+	}
+
+	void syscall_failed(string const &call, string const &message)
+	{
+		ostringstream out;
+		out << "syscall '" << call << "' failed: " << base::error_string(errno) << "\n"
+		    << message;
 		throw runtime_error(out.str());
 	}
 
 	int open_file(string const &path, int flags) {
 		int fd = ::open(path.c_str(), OPEN_FLAGS | flags, DEFAULT_MODE);
 		if (fd < 0)
-			syscall_failed("open");
+			syscall_failed("open",
+				 "Note: you cannot run this tool with these options on live metadata.");
 
 		return fd;
 	}
@@ -80,7 +89,7 @@ namespace {
 			throw runtime_error(out.str());
 		}
 
-		int fd = open_file(path, O_CREAT | O_RDWR);
+		int fd = open_file(path, O_CREAT | O_EXCL | O_RDWR);
 
 		int r = ::ftruncate(fd, file_size);
 		if (r < 0)
@@ -89,14 +98,18 @@ namespace {
 		return fd;
 	}
 
-	int open_block_file(string const &path, off_t min_size, bool writeable) {
+	int open_block_file(string const &path, off_t min_size, bool writeable, bool excl = true) {
 		if (!file_exists(path)) {
 			ostringstream out;
 			out << __FUNCTION__ << ": file '" << path << "' doesn't exist";
 			throw runtime_error(out.str());
 		}
 
-		return open_file(path, writeable ? O_RDWR : O_RDONLY);
+		int flags = writeable ? O_RDWR : O_RDONLY;
+		if (excl)
+			flags |= O_EXCL;
+
+		return open_file(path, flags);
 	}
 };
 
@@ -208,8 +221,9 @@ namespace persistent_data {
 	block_manager<BlockSize>::block_manager(std::string const &path,
 						block_address nr_blocks,
 						unsigned max_concurrent_blocks,
-						mode m)
-		: fd_(open_or_create_block_file(path, nr_blocks * BlockSize, m)),
+						mode m,
+						bool excl)
+		: fd_(open_or_create_block_file(path, nr_blocks * BlockSize, m, excl)),
 		  bc_(fd_, BlockSize >> SECTOR_SHIFT, nr_blocks, 1024u * 1024u * 16),
 		  superblock_ref_count_(0)
 	{
@@ -217,14 +231,14 @@ namespace persistent_data {
 
 	template <uint32_t BlockSize>
 	int
-	block_manager<BlockSize>::open_or_create_block_file(string const &path, off_t file_size, mode m)
+	block_manager<BlockSize>::open_or_create_block_file(string const &path, off_t file_size, mode m, bool excl)
 	{
 		switch (m) {
 		case READ_ONLY:
-			return open_block_file(path, file_size, false);
+			return open_block_file(path, file_size, false, excl);
 
 		case READ_WRITE:
-			return open_block_file(path, file_size, true);
+			return open_block_file(path, file_size, true, excl);
 
 		case CREATE:
 			return create_block_file(path, file_size);

persistent-data/data-structures/btree.h

@@ -22,6 +22,7 @@
 #include "base/endian_utils.h"
 #include "persistent-data/transaction_manager.h"
 #include "persistent-data/data-structures/ref_counter.h"
+#include "persistent-data/data-structures/btree_disk_structures.h"
 
 #include <boost/noncopyable.hpp>
 #include <boost/optional.hpp>
@@ -61,36 +62,6 @@ namespace persistent_data {
 		using namespace base;
 		using namespace std;
 
-		uint32_t const BTREE_CSUM_XOR = 121107;
-
-		//------------------------------------------------
-		// On disk data layout for btree nodes
-		enum node_flags {
-			INTERNAL_NODE = 1,
-			LEAF_NODE = 1 << 1
-		};
-
-		struct node_header {
-			le32 csum;
-			le32 flags;
-			le64 blocknr; /* which block this node is supposed to live in */
-
-			le32 nr_entries;
-			le32 max_entries;
-			le32 value_size;
-			le32 padding;
-		} __attribute__((packed));
-
-		struct disk_node {
-			struct node_header header;
-			le64 keys[0];
-		} __attribute__((packed));
-
-		enum node_type {
-			INTERNAL,
-			LEAF
-		};
-
 		//------------------------------------------------
 		// Class that acts as an interface over the raw little endian btree
 		// node data.
@@ -161,6 +132,9 @@ namespace persistent_data {
 				return raw_;
 			}
 
+			bool value_sizes_match() const;
+			std::string value_mismatch_string() const;
+
 		private:
 			static unsigned calc_max_entries(void);
 			void check_fits_within_block() const;

persistent-data/data-structures/btree.tcc

@@ -21,6 +21,7 @@
 #include "persistent-data/errors.h"
 #include "persistent-data/checksum.h"
 #include "persistent-data/transaction_manager.h"
+#include "persistent-data/validators.h"
 
 #include <iostream>
 
@@ -32,29 +33,6 @@ namespace {
 	using namespace btree_detail;
 	using namespace std;
 
-	struct btree_node_validator : public bcache::validator {
-		virtual void check(void const *raw, block_address location) const {
-			disk_node const *data = reinterpret_cast<disk_node const *>(raw);
-			node_header const *n = &data->header;
-			crc32c sum(BTREE_CSUM_XOR);
-			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
-			if (sum.get_sum() != to_cpu<uint32_t>(n->csum))
-				throw checksum_error("bad checksum in btree node");
-
-			if (to_cpu<uint64_t>(n->blocknr) != location)
-				throw checksum_error("bad block nr in btree node");
-		}
-
-		virtual void prepare(void *raw, block_address location) const {
-			disk_node *data = reinterpret_cast<disk_node *>(raw);
-			node_header *n = &data->header;
-			n->blocknr = to_disk<base::le64, uint64_t>(location);
-
-			crc32c sum(BTREE_CSUM_XOR);
-			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
-			n->csum = to_disk<base::le32>(sum.get_sum());
-		}
-	};
 }
 
 //----------------------------------------------------------------
@@ -362,19 +340,31 @@ namespace persistent_data {
 		}
 	}
 
+	template <typename ValueTraits>
+	bool
+	node_ref<ValueTraits>::value_sizes_match() const {
+		return sizeof(typename ValueTraits::disk_type) == get_value_size();
+	}
+
+	template <typename ValueTraits>
+	std::string
+	node_ref<ValueTraits>::value_mismatch_string() const {
+		std::ostringstream out;
+		out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type)
+		    << ", but got " << get_value_size()
+		    << ".  This is not the btree you are looking for." << std::endl;
+
+		return out.str();
+	}
+
 	template <typename ValueTraits>
 	void
 	node_ref<ValueTraits>::check_fits_within_block() const {
 		if (checked_)
 			return;
 
-		if (sizeof(typename ValueTraits::disk_type) != get_value_size()) {
-			std::ostringstream out;
-			out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type)
-			    << ", but got " << get_value_size()
-			    << ".  This is not the btree you are looking for." << std::endl;
-			throw std::runtime_error(out.str());
-		}
+		if (!value_sizes_match())
+			throw std::runtime_error(value_mismatch_string());
 
 		unsigned max = calc_max_entries();
 
@@ -398,7 +388,7 @@ namespace persistent_data {
 		  destroy_(false),
 		  internal_rc_(tm.get_sm()),
 		  rc_(rc),
-		  validator_(new btree_node_validator)
+		  validator_(create_btree_node_validator())
 	{
 		using namespace btree_detail;
 
@@ -432,7 +422,7 @@ namespace persistent_data {
 		  root_(root),
 		  internal_rc_(tm.get_sm()),
 		  rc_(rc),
-		  validator_(new btree_node_validator)
+		  validator_(create_btree_node_validator())
 	{
 	}
 
@@ -446,7 +436,8 @@ namespace persistent_data {
 		template <typename ValueTraits>
 		struct lower_bound_search {
 			static boost::optional<unsigned> search(btree_detail::node_ref<ValueTraits> n, uint64_t key) {
-				return n.lower_bound(key);
+				int i = n.lower_bound(key);
+				return (i < 0) ? boost::optional<unsigned>() : boost::optional<unsigned>(i);
 			}
 		};
 
@@ -595,9 +586,13 @@ namespace persistent_data {
 
 			}
 
-			mi = leaf.lower_bound(key);
-			if (!mi || *mi < 0)
-				return boost::optional<leaf_type>();
+			{
+				int lb = leaf.lower_bound(key);
+				if (lb < 0)
+					return boost::optional<leaf_type>();
+
+				mi = lb;
+			}
 
 			node_ref<block_traits> internal = spine.template get_node<block_traits>();
 			block = internal.value_at(*mi);

persistent-data/data-structures/btree_damage_visitor.h

@@ -70,10 +70,17 @@ namespace persistent_data {
 			}
 
 			maybe_run64 end() {
+				maybe_run64 r;
+
 				if (damaged_)
-					return maybe_run64(damage_begin_);
+					r = maybe_run64(damage_begin_);
 				else
-					return maybe_run64();
+					r = maybe_run64();
+
+				damaged_ = false;
+				damage_begin_ = 0;
+
+				return r;
 			}
 
 		private:
@@ -190,6 +197,7 @@ namespace persistent_data {
 
 			error_outcome error_accessing_node(node_location const &l, block_address b,
 							   std::string const &what) {
+				update_path(l.path);
 				report_damage(what);
 				return btree<Levels, ValueTraits>::visitor::EXCEPTION_HANDLED;
 			}
@@ -210,6 +218,7 @@ namespace persistent_data {
 					    btree_detail::node_ref<block_traits> const &n) {
 				if (!already_visited(n) &&
 				    check_block_nr(n) &&
+				    check_value_size(n) &&
 				    check_max_entries(n) &&
 				    check_nr_entries(n, loc.is_sub_root()) &&
 				    check_ordered_keys(n) &&
@@ -229,6 +238,7 @@ namespace persistent_data {
 					btree_detail::node_ref<ValueTraits2> const &n) {
 				if (!already_visited(n) &&
 				    check_block_nr(n) &&
+				    check_value_size(n) &&
 				    check_max_entries(n) &&
 				    check_nr_entries(n, loc.is_sub_root()) &&
 				    check_ordered_keys(n) &&
@@ -275,6 +285,16 @@ namespace persistent_data {
 				return true;
 			}
 
+			template <typename node>
+			bool check_value_size(node const &n) {
+				if (!n.value_sizes_match()) {
+					report_damage(n.value_mismatch_string());
+					return false;
+				}
+
+				return true;
+			}
+
 			template <typename node>
 			bool check_max_entries(node const &n) {
 				size_t elt_size = sizeof(uint64_t) + n.get_value_size();

persistent-data/data-structures/btree_disk_structures.h

@@ -0,0 +1,64 @@
+// Copyright (C) 2011 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef PERSISTENT_DATA_BTREE_DISK_STRUCTURES_H
+#define PERSISTENT_DATA_BTREE_DISK_STRUCTURES_H
+
+#include "base/endian_utils.h"
+
+//----------------------------------------------------------------
+
+namespace persistent_data {
+	namespace btree_detail {
+		using namespace base;
+
+		uint32_t const BTREE_CSUM_XOR = 121107;
+
+		//------------------------------------------------
+		// On disk data layout for btree nodes
+		enum node_flags {
+			INTERNAL_NODE = 1,
+			LEAF_NODE = 1 << 1
+		};
+
+		struct node_header {
+			le32 csum;
+			le32 flags;
+			le64 blocknr; /* which block this node is supposed to live in */
+
+			le32 nr_entries;
+			le32 max_entries;
+			le32 value_size;
+			le32 padding;
+		} __attribute__((packed));
+
+		struct disk_node {
+			struct node_header header;
+			le64 keys[0];
+		} __attribute__((packed));
+
+		enum node_type {
+			INTERNAL,
+			LEAF
+		};
+	}
+}
+
+//----------------------------------------------------------------
+
+#endif

persistent-data/file_utils.cc

@@ -48,10 +48,10 @@ persistent_data::get_nr_blocks(string const &path)
 }
 
 persistent_data::block_manager<>::ptr
-persistent_data::open_bm(std::string const &dev_path, block_manager<>::mode m)
+persistent_data::open_bm(std::string const &dev_path, block_manager<>::mode m, bool excl)
 {
 	block_address nr_blocks = get_nr_blocks(dev_path);
-	return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m));
+	return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m, excl));
 }
 
 void

persistent-data/file_utils.h

@@ -10,7 +10,8 @@
 // FIXME: move to a different unit
 namespace persistent_data {
 	persistent_data::block_address get_nr_blocks(string const &path);
-	block_manager<>::ptr open_bm(std::string const &dev_path, block_manager<>::mode m);
+	block_manager<>::ptr open_bm(std::string const &dev_path,
+				     block_manager<>::mode m, bool excl = true);
 
 	void check_file_exists(std::string const &file);
 }

persistent-data/space-maps/disk.cc

@@ -747,6 +747,11 @@ persistent_data::create_metadata_sm(transaction_manager &tm, block_address nr_bl
 {
 	index_store::ptr store(new metadata_index_store(tm));
 	checked_space_map::ptr sm(new sm_disk(store, tm));
+
+	if (nr_blocks > MAX_METADATA_BLOCKS) {
+		cerr << "truncating metadata device to " << MAX_METADATA_BLOCKS << " 4k blocks\n";
+		nr_blocks = MAX_METADATA_BLOCKS;
+	}
 	sm->extend(nr_blocks);
 	sm->commit();
 	return create_careful_alloc_sm(

persistent-data/space-maps/disk_structures.h

@@ -61,6 +61,7 @@ namespace persistent_data {
 		};
 
 		unsigned const MAX_METADATA_BITMAPS = 255;
+		unsigned const MAX_METADATA_BLOCKS = (255 * ((1 << 14) - 64));
 		unsigned const ENTRIES_PER_BYTE = 4;
 
 		struct metadata_index {

persistent-data/validators.cc

@@ -0,0 +1,53 @@
+#include "persistent-data/block.h"
+#include "persistent-data/checksum.h"
+#include "persistent-data/data-structures/btree_disk_structures.h"
+#include "persistent-data/errors.h"
+#include "persistent-data/validators.h"
+
+using namespace bcache;
+using namespace persistent_data;
+
+//----------------------------------------------------------------
+
+namespace {
+	using namespace btree_detail;
+
+	struct btree_node_validator : public bcache::validator {
+		virtual void check(void const *raw, block_address location) const {
+			disk_node const *data = reinterpret_cast<disk_node const *>(raw);
+			node_header const *n = &data->header;
+			crc32c sum(BTREE_CSUM_XOR);
+			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
+			if (sum.get_sum() != to_cpu<uint32_t>(n->csum)) {
+				std::ostringstream out;
+				out << "bad checksum in btree node (block " << location << ")";
+				throw checksum_error(out.str());
+			}
+
+			if (to_cpu<uint64_t>(n->blocknr) != location) {
+				std::ostringstream out;
+				out << "bad block nr in btree node (block = " << location << ")";
+				throw checksum_error(out.str());
+			}
+		}
+
+		virtual void prepare(void *raw, block_address location) const {
+			disk_node *data = reinterpret_cast<disk_node *>(raw);
+			node_header *n = &data->header;
+			n->blocknr = to_disk<base::le64, uint64_t>(location);
+
+			crc32c sum(BTREE_CSUM_XOR);
+			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
+			n->csum = to_disk<base::le32>(sum.get_sum());
+		}
+	};
+}
+
+//----------------------------------------------------------------
+
+bcache::validator::ptr persistent_data::create_btree_node_validator()
+{
+	return bcache::validator::ptr(new btree_node_validator());
+}
+
+//----------------------------------------------------------------

persistent-data/validators.h

@@ -0,0 +1,14 @@
+#ifndef PERSISTENT_DATA_VALIDATORS_H
+#define PERSISTENT_DATA_VALIDATORS_H
+
+#include "block-cache/block_cache.h"
+
+//----------------------------------------------------------------
+
+namespace persistent_data {
+	bcache::validator::ptr create_btree_node_validator();
+}
+
+//----------------------------------------------------------------
+
+#endif

thin-provisioning/commands.cc

@@ -0,0 +1,22 @@
+#include "thin-provisioning/commands.h"
+
+using namespace base;
+using namespace thin_provisioning;
+
+//----------------------------------------------------------------
+
+void
+thin_provisioning::register_thin_commands(base::application &app)
+{
+	app.add_cmd(command::ptr(new thin_check_cmd()));
+	app.add_cmd(command::ptr(new thin_delta_cmd()));
+	app.add_cmd(command::ptr(new thin_dump_cmd()));
+	app.add_cmd(command::ptr(new thin_ls_cmd()));
+	app.add_cmd(command::ptr(new thin_metadata_size_cmd()));
+	app.add_cmd(command::ptr(new thin_restore_cmd()));
+	app.add_cmd(command::ptr(new thin_repair_cmd()));
+	app.add_cmd(command::ptr(new thin_rmap_cmd()));
+	app.add_cmd(command::ptr(new thin_trim_cmd()));
+}
+
+//----------------------------------------------------------------

thin-provisioning/commands.h

@@ -2,19 +2,75 @@
 #define THIN_PROVISIONING_COMMANDS_H
 
 #include "base/application.h"
+#include "boost/optional.hpp"
 
 //----------------------------------------------------------------
 
 namespace thin_provisioning {
-	extern base::command thin_check_cmd;
-	extern base::command thin_delta_cmd;
-	extern base::command thin_dump_cmd;
-	extern base::command thin_metadata_size_cmd;
-	extern base::command thin_restore_cmd;
-	extern base::command thin_repair_cmd;
-	extern base::command thin_rmap_cmd;
-	extern base::command thin_trim_cmd;
-	extern base::command thin_metadata_size_cmd;
+	class thin_check_cmd : public base::command {
+	public:
+		thin_check_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_delta_cmd : public base::command {
+	public:
+		thin_delta_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_dump_cmd : public base::command {
+	public:
+		thin_dump_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_ls_cmd : public base::command {
+	public:
+		thin_ls_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_metadata_size_cmd : public base::command {
+	public:
+		thin_metadata_size_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_restore_cmd : public base::command {
+	public:
+		thin_restore_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_repair_cmd : public base::command {
+	public:
+		thin_repair_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_rmap_cmd : public base::command {
+	public:
+		thin_rmap_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_trim_cmd : public base::command {
+	public:
+		thin_trim_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	void register_thin_commands(base::application &app);
 }
 
 //----------------------------------------------------------------

thin-provisioning/mapping_tree.cc

@@ -178,6 +178,10 @@ namespace {
 				v_.visit(missing_devices(d.desc_, d.lost_keys_));
 				break;
 
+			case 1:
+				v_.visit(missing_mappings(d.desc_, path[0], d.lost_keys_));
+				break;
+
 			default:
 				throw std::runtime_error("mapping_tree_damage_visitor: path too long");
 			}
@@ -196,7 +200,7 @@ namespace {
 		virtual void visit(btree_path const &path, btree_detail::damage const &d) {
 			switch (path.size()) {
 			case 0:
-				v_.visit(missing_mappings(d.desc_, path[0], d.lost_keys_));
+				v_.visit(missing_devices(d.desc_, d.lost_keys_));
 				break;
 
 			default:

thin-provisioning/metadata.cc

@@ -60,80 +60,6 @@ namespace {
 
 //----------------------------------------------------------------
 
-metadata::metadata(std::string const &dev_path, open_type ot,
-		   sector_t data_block_size, block_address nr_data_blocks)
-{
-	switch (ot) {
-	case OPEN:
-		tm_ = open_tm(open_bm(dev_path, block_manager<>::READ_ONLY));
-		sb_ = read_superblock(tm_->get_bm());
-
-		if (sb_.version_ != 1)
-			throw runtime_error("unknown metadata version");
-
-		metadata_sm_ = open_metadata_sm(*tm_, &sb_.metadata_space_map_root_);
-		tm_->set_sm(metadata_sm_);
-
-		data_sm_ = open_disk_sm(*tm_, static_cast<void *>(&sb_.data_space_map_root_));
-
-		details_ = device_tree::ptr(
-			new device_tree(*tm_, sb_.device_details_root_,
-					device_tree_detail::device_details_traits::ref_counter()));
-
-		mappings_top_level_ = dev_tree::ptr(
-			new dev_tree(*tm_, sb_.data_mapping_root_,
-				     mapping_tree_detail::mtree_ref_counter(tm_)));
-
-		mappings_ = mapping_tree::ptr(
-			new mapping_tree(*tm_, sb_.data_mapping_root_,
-					 mapping_tree_detail::block_time_ref_counter(data_sm_)));
-		break;
-
-	case CREATE:
-		tm_ = open_tm(open_bm(dev_path, block_manager<>::READ_WRITE));
-		space_map::ptr core = tm_->get_sm();
-		metadata_sm_ = create_metadata_sm(*tm_, tm_->get_bm()->get_nr_blocks());
-		copy_space_maps(metadata_sm_, core);
-		tm_->set_sm(metadata_sm_);
-
-		data_sm_ = create_disk_sm(*tm_, nr_data_blocks);
-		details_ = device_tree::ptr(new device_tree(*tm_, device_tree_detail::device_details_traits::ref_counter()));
-		mappings_ = mapping_tree::ptr(new mapping_tree(*tm_,
-							       mapping_tree_detail::block_time_ref_counter(data_sm_)));
-		mappings_top_level_ = dev_tree::ptr(new dev_tree(*tm_, mappings_->get_root(),
-								 mapping_tree_detail::mtree_ref_counter(tm_)));
-
-		::memset(&sb_, 0, sizeof(sb_));
-		sb_.magic_ = SUPERBLOCK_MAGIC;
-		sb_.version_ = 1;
-		sb_.data_mapping_root_ = mappings_->get_root();
-		sb_.device_details_root_ = details_->get_root();
-		sb_.data_block_size_ = data_block_size;
-		sb_.metadata_block_size_ = MD_BLOCK_SIZE;
-		sb_.metadata_nr_blocks_ = tm_->get_bm()->get_nr_blocks();
-
-		break;
-	}
-}
-
-metadata::metadata(std::string const &dev_path, block_address metadata_snap)
-{
-	tm_ = open_tm(open_bm(dev_path, block_manager<>::READ_ONLY));
-	sb_ = read_superblock(tm_->get_bm(), metadata_snap);
-
-	// We don't open the metadata sm for a held root
-	//metadata_sm_ = open_metadata_sm(tm_, &sb_.metadata_space_map_root_);
-	//tm_->set_sm(metadata_sm_);
-
-	data_sm_ = open_disk_sm(*tm_, static_cast<void *>(&sb_.data_space_map_root_));
-	details_ = device_tree::ptr(new device_tree(*tm_, sb_.device_details_root_, device_tree_detail::device_details_traits::ref_counter()));
-	mappings_top_level_ = dev_tree::ptr(new dev_tree(*tm_, sb_.data_mapping_root_,
-							 mapping_tree_detail::mtree_ref_counter(tm_)));
-	mappings_ = mapping_tree::ptr(new mapping_tree(*tm_, sb_.data_mapping_root_,
-						       mapping_tree_detail::block_time_ref_counter(data_sm_)));
-}
-
-// FIXME: duplication
 metadata::metadata(block_manager<>::ptr bm, open_type ot,
 		   sector_t data_block_size,
 		   block_address nr_data_blocks)
@@ -150,7 +76,8 @@ metadata::metadata(block_manager<>::ptr bm, open_type ot,
 		tm_->set_sm(metadata_sm_);
 
 		data_sm_ = open_disk_sm(*tm_, static_cast<void *>(&sb_.data_space_map_root_));
-		details_ = device_tree::ptr(new device_tree(*tm_, sb_.device_details_root_, device_tree_detail::device_details_traits::ref_counter()));
+		details_ = device_tree::ptr(new device_tree(*tm_, sb_.device_details_root_,
+							    device_tree_detail::device_details_traits::ref_counter()));
 		mappings_top_level_ = dev_tree::ptr(new dev_tree(*tm_, sb_.data_mapping_root_,
 								 mapping_tree_detail::mtree_ref_counter(tm_)));
 		mappings_ = mapping_tree::ptr(new mapping_tree(*tm_, sb_.data_mapping_root_,
@@ -165,7 +92,8 @@ metadata::metadata(block_manager<>::ptr bm, open_type ot,
 		tm_->set_sm(metadata_sm_);
 
 		data_sm_ = create_disk_sm(*tm_, nr_data_blocks);
-		details_ = device_tree::ptr(new device_tree(*tm_, device_tree_detail::device_details_traits::ref_counter()));
+		details_ = device_tree::ptr(new device_tree(*tm_,
+							    device_tree_detail::device_details_traits::ref_counter()));
 		mappings_ = mapping_tree::ptr(new mapping_tree(*tm_,
 							       mapping_tree_detail::block_time_ref_counter(data_sm_)));
 		mappings_top_level_ = dev_tree::ptr(new dev_tree(*tm_, mappings_->get_root(),
@@ -177,13 +105,41 @@ metadata::metadata(block_manager<>::ptr bm, open_type ot,
 		sb_.data_mapping_root_ = mappings_->get_root();
 		sb_.device_details_root_ = details_->get_root();
 		sb_.data_block_size_ = data_block_size;
-		sb_.metadata_block_size_ = MD_BLOCK_SIZE;
+		sb_.metadata_block_size_ = MD_BLOCK_SIZE >> SECTOR_SHIFT;
 		sb_.metadata_nr_blocks_ = tm_->get_bm()->get_nr_blocks();
 
 		break;
 	}
 }
 
+metadata::metadata(block_manager<>::ptr bm)
+{
+	tm_ = open_tm(bm);
+	sb_ = read_superblock(tm_->get_bm(), SUPERBLOCK_LOCATION);
+
+	open_space_maps();
+	open_btrees();
+}
+
+metadata::metadata(block_manager<>::ptr bm,
+		   boost::optional<block_address> metadata_snap)
+{
+	tm_ = open_tm(bm);
+
+	superblock_detail::superblock actual_sb = read_superblock(bm, SUPERBLOCK_LOCATION);
+
+	if (!actual_sb.metadata_snap_)
+		throw runtime_error("no current metadata snap");
+
+	if (metadata_snap && *metadata_snap != actual_sb.metadata_snap_)
+			throw runtime_error("metadata snapshot does not match that in superblock");
+
+	sb_ = read_superblock(bm, actual_sb.metadata_snap_);
+
+	// metadata snaps don't record the space maps
+	open_btrees();
+}
+
 void
 metadata::commit()
 {
@@ -201,4 +157,22 @@ metadata::commit()
 	superblock_traits::pack(sb_, *disk);
 }
 
+void metadata::open_space_maps()
+{
+	metadata_sm_ = open_metadata_sm(*tm_, &sb_.metadata_space_map_root_);
+	tm_->set_sm(metadata_sm_);
+
+	data_sm_ = open_disk_sm(*tm_, static_cast<void *>(&sb_.data_space_map_root_));
+}
+
+void metadata::open_btrees()
+{
+	details_ = device_tree::ptr(new device_tree(*tm_, sb_.device_details_root_,
+						    device_tree_detail::device_details_traits::ref_counter()));
+	mappings_top_level_ = dev_tree::ptr(new dev_tree(*tm_, sb_.data_mapping_root_,
+							 mapping_tree_detail::mtree_ref_counter(tm_)));
+	mappings_ = mapping_tree::ptr(new mapping_tree(*tm_, sb_.data_mapping_root_,
+						       mapping_tree_detail::block_time_ref_counter(data_sm_)));
+}
+
 //----------------------------------------------------------------

thin-provisioning/metadata.h

@@ -59,21 +59,24 @@ namespace thin_provisioning {
 		typedef block_manager<>::write_ref write_ref;
 		typedef boost::shared_ptr<metadata> ptr;
 
-
-		// Deprecated: it would be better if we passed in an already
-		// constructed block_manager.
-		metadata(std::string const &dev_path, open_type ot,
-			 sector_t data_block_size = 128, // Only used if CREATE
-			 block_address nr_data_blocks = 0); // Only used if CREATE
-
-		metadata(std::string const &dev_path,
-			 block_address metadata_snap = 0);
-
-		// ... use these instead ...
 		metadata(block_manager<>::ptr bm, open_type ot,
 			 sector_t data_block_size = 128,
 			 block_address nr_data_blocks = 0); // Only used if CREATE
-		metadata(block_manager<>::ptr bm, block_address metadata_snap);
+
+		// Ideally we'd like the metadata snap argument to be a
+		// boolean, and we'd read the snap location from the
+		// superblock.  But the command line interface for some of
+		// the tools allows the user to pass in a block, which
+		// they've retrieved from the pool status.  So we have to
+		// support 3 cases:
+		//
+		// i)   Read superblock
+		// ii)  Read the metadata snap as given in the superblock
+		// iii) Read the metadata snap given on command line, checking it matches superblock.
+		//
+		metadata(block_manager<>::ptr bm); // (i)
+		metadata(block_manager<>::ptr,
+			 boost::optional<block_address> metadata_snap); // (ii) and (iii)
 
 		void commit();
 
@@ -86,6 +89,10 @@ namespace thin_provisioning {
 		device_tree::ptr details_;
 		dev_tree::ptr mappings_top_level_;
 		mapping_tree::ptr mappings_;
+
+	private:
+		void open_space_maps();
+		void open_btrees();
 	};
 }
 

thin-provisioning/metadata_dumper.cc

@@ -84,7 +84,7 @@ namespace {
 
 	//--------------------------------
 
-	typedef map<block_address, device_tree_detail::device_details> dd_map;
+ 	typedef map<block_address, device_tree_detail::device_details> dd_map;
 
 	class details_extractor : public device_tree_detail::device_visitor {
 	public:

thin-provisioning/restore_emitter.cc

@@ -50,7 +50,8 @@ namespace {
 			in_superblock_ = true;
 			nr_data_blocks_ = nr_data_blocks;
 			superblock &sb = md_->sb_;
-			memcpy(&sb.uuid_, &uuid, sizeof(sb.uuid_));
+			memset(&sb.uuid_, 0, sizeof(sb.uuid_));
+			memcpy(&sb.uuid_, uuid.c_str(), std::min(sizeof(sb.uuid_), uuid.length()));
 			sb.time_ = time;
 			sb.trans_id_ = trans_id;
 			sb.data_block_size_ = data_block_size;

thin-provisioning/superblock.cc

@@ -1,6 +1,7 @@
 #include "persistent-data/checksum.h"
 #include "persistent-data/errors.h"
 #include "thin-provisioning/superblock.h"
+#include "persistent-data/file_utils.h"
 
 using namespace thin_provisioning;
 using namespace superblock_detail;

thin-provisioning/thin_check.cc

@@ -169,23 +169,9 @@ namespace {
 		bool clear_needs_check_flag_on_success;
 	};
 
-	error_state check_space_map_counts(flags const &fs, nested_output &out,
-					   superblock_detail::superblock &sb,
-					   block_manager<>::ptr bm,
-					   transaction_manager::ptr tm) {
-		block_counter bc;
-
-		// Count the superblock
-		bc.inc(superblock_detail::SUPERBLOCK_LOCATION);
-
-		// Count the metadata snap, if present
-		if (sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) {
-			bc.inc(sb.metadata_snap_);
-
-			superblock_detail::superblock snap = read_superblock(bm, sb.metadata_snap_);
-			bc.inc(snap.data_mapping_root_);
-			bc.inc(snap.device_details_root_);
-		}
+	void count_trees(transaction_manager::ptr tm,
+			 superblock_detail::superblock &sb,
+			 block_counter &bc) {
 
 		// Count the device tree
 		{
@@ -202,6 +188,25 @@ namespace {
 					   mapping_tree_detail::block_traits::ref_counter(tm->get_sm()));
 			count_btree_blocks(mtree, bc, vc);
 		}
+	}
+
+	error_state check_space_map_counts(flags const &fs, nested_output &out,
+					   superblock_detail::superblock &sb,
+					   block_manager<>::ptr bm,
+					   transaction_manager::ptr tm) {
+		block_counter bc;
+
+		// Count the superblock
+		bc.inc(superblock_detail::SUPERBLOCK_LOCATION);
+		count_trees(tm, sb, bc);
+
+		// Count the metadata snap, if present
+		if (sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) {
+			bc.inc(sb.metadata_snap_);
+
+			superblock_detail::superblock snap = read_superblock(bm, sb.metadata_snap_);
+			count_trees(tm, snap, bc);
+		}
 
 		// Count the metadata space map
 		{
@@ -323,11 +328,11 @@ namespace {
 			err = metadata_check(path, fs);
 
 			if (fs.ignore_non_fatal_errors)
-				success = (err == FATAL) ? 1 : 0;
+				success = (err == FATAL) ? false : true;
 			else
-				success =  (err == NO_ERROR) ? 0 : 1;
+				success = (err == NO_ERROR) ? true : false;
 
-			if (!success && fs.clear_needs_check_flag_on_success)
+			if (success && fs.clear_needs_check_flag_on_success)
 				clear_needs_check(path);
 
 		} catch (std::exception &e) {
@@ -337,23 +342,33 @@ namespace {
 			return 1;
 		}
 
-		return success;
-	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-q|--quiet}" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--clear-needs-check-flag}" << endl
-		    << "  {--ignore-non-fatal-errors}" << endl
-		    << "  {--skip-mappings}" << endl
-		    << "  {--super-block-only}" << endl;
+		return !success;
 	}
 }
 
-int thin_check_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+thin_check_cmd::thin_check_cmd()
+	: command("thin_check")
+{
+}
+
+void
+thin_check_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-q|--quiet}" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--clear-needs-check-flag}" << endl
+	    << "  {--ignore-non-fatal-errors}" << endl
+	    << "  {--skip-mappings}" << endl
+	    << "  {--super-block-only}" << endl;
+}
+
+int
+thin_check_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -373,7 +388,7 @@ int thin_check_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
 		switch(c) {
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'q':
@@ -407,7 +422,7 @@ int thin_check_main(int argc, char **argv)
 			break;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
@@ -415,7 +430,7 @@ int thin_check_main(int argc, char **argv)
 	if (argc == optind) {
 		if (!fs.quiet) {
 			cerr << "No input file provided." << endl;
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 		}
 
 		exit(1);
@@ -424,6 +439,4 @@ int thin_check_main(int argc, char **argv)
 	return check(argv[optind], fs);
 }
 
-base::command thin_provisioning::thin_check_cmd("thin_check", thin_check_main);
-
 //----------------------------------------------------------------

thin-provisioning/thin_delta.cc

@@ -14,6 +14,7 @@
 #include "persistent-data/file_utils.h"
 #include "thin-provisioning/superblock.h"
 #include "thin-provisioning/mapping_tree.h"
+#include "thin-provisioning/metadata.h"
 #include "thin-provisioning/commands.h"
 
 using namespace std;
@@ -64,25 +65,21 @@ namespace local {
 
 	struct flags {
 		flags()
-			: verbose(false) {
+			: verbose(false),
+			  use_metadata_snap(false) {
 		}
 
+		bool verbose;
+		bool use_metadata_snap;
+
 		boost::optional<string> dev;
 		boost::optional<uint64_t> metadata_snap;
 		boost::optional<uint64_t> snap1;
 		boost::optional<uint64_t> snap2;
-		bool verbose;
 	};
 
 	//--------------------------------
 
-	block_manager<>::ptr
-	open_bm(string const &path) {
-		block_address nr_blocks = get_nr_blocks(path);
-		block_manager<>::mode m = block_manager<>::READ_ONLY;
-		return block_manager<>::ptr(new block_manager<>(path, nr_blocks, 1, m));
-	}
-
 	transaction_manager::ptr
 	open_tm(block_manager<>::ptr bm) {
 		space_map::ptr sm(new core_map(bm->get_nr_blocks()));
@@ -533,17 +530,12 @@ namespace local {
 		checked_space_map::ptr data_sm;
 
 		{
-			block_manager<>::ptr bm = open_bm(*fs.dev);
-			transaction_manager::ptr tm = open_tm(bm);
-
-			sb = fs.metadata_snap ? read_superblock(bm, *fs.metadata_snap) : read_superblock(bm);
-			data_sm = open_disk_sm(*tm, static_cast<void *>(&sb.data_space_map_root_));
-
-			dev_tree dtree(*tm, sb.data_mapping_root_,
-				       mapping_tree_detail::mtree_traits::ref_counter(tm));
+			block_manager<>::ptr bm = open_bm(*fs.dev, block_manager<>::READ_ONLY, !fs.use_metadata_snap);
+			metadata::ptr md(fs.use_metadata_snap ? new metadata(bm, fs.metadata_snap) : new metadata(bm));
+			sb = md->sb_;
 
 			dev_tree::key k = {*fs.snap1};
-			boost::optional<uint64_t> snap1_root = dtree.lookup(k);
+			boost::optional<uint64_t> snap1_root = md->mappings_top_level_->lookup(k);
 
 			if (!snap1_root) {
 				ostringstream out;
@@ -551,10 +543,11 @@ namespace local {
 				app.die(out.str());
 			}
 
-			single_mapping_tree snap1(*tm, *snap1_root, mapping_tree_detail::block_traits::ref_counter(tm->get_sm()));
+			single_mapping_tree snap1(*md->tm_, *snap1_root,
+						  mapping_tree_detail::block_traits::ref_counter(md->tm_->get_sm()));
 
 			k[0] = *fs.snap2;
-			boost::optional<uint64_t> snap2_root = dtree.lookup(k);
+			boost::optional<uint64_t> snap2_root = md->mappings_top_level_->lookup(k);
 
 			if (!snap2_root) {
 				ostringstream out;
@@ -562,7 +555,8 @@ namespace local {
 				app.die(out.str());
 			}
 
-			single_mapping_tree snap2(*tm, *snap2_root, mapping_tree_detail::block_traits::ref_counter(tm->get_sm()));
+			single_mapping_tree snap2(*md->tm_, *snap2_root,
+						  mapping_tree_detail::block_traits::ref_counter(md->tm_->get_sm()));
 			btree_visit_values(snap1, mr1, damage_v);
 			mr1.complete();
 
@@ -608,7 +602,19 @@ namespace local {
 
 // FIXME: add metadata snap switch
 
-int thin_delta_main(int argc, char **argv)
+thin_delta_cmd::thin_delta_cmd()
+	: command("thin_delta")
+{
+}
+
+void
+thin_delta_cmd::usage(std::ostream &out) const
+{
+	// FIXME: finish
+}
+
+int
+thin_delta_cmd::run(int argc, char **argv)
 {
 	using namespace local;
 
@@ -616,7 +622,7 @@ int thin_delta_main(int argc, char **argv)
 	flags fs;
 	local::application app(basename(argv[0]));
 
-	char const shortopts[] = "hVm";
+	char const shortopts[] = "hVm::";
 	option const longopts[] = {
 		{ "help", no_argument, NULL, 'h' },
 		{ "version", no_argument, NULL, 'V' },
@@ -624,7 +630,7 @@ int thin_delta_main(int argc, char **argv)
 		{ "snap1", required_argument, NULL, 1 },
 		{ "thin2", required_argument, NULL, 2 },
 		{ "snap2", required_argument, NULL, 2 },
-		{ "metadata-snap", no_argument, NULL, 'm' },
+		{ "metadata-snap", optional_argument, NULL, 'm' },
 		{ "verbose", no_argument, NULL, 4 }
 	};
 
@@ -647,7 +653,9 @@ int thin_delta_main(int argc, char **argv)
 			break;
 
 		case 'm':
-			fs.metadata_snap = app.parse_int(optarg, "metadata snapshot block");
+			fs.use_metadata_snap = true;
+			if (optarg)
+				fs.metadata_snap = app.parse_int(optarg, "metadata snapshot block");
 			break;
 
 		case 4:
@@ -674,6 +682,4 @@ int thin_delta_main(int argc, char **argv)
 	return delta(app, fs);
 }
 
-base::command thin_provisioning::thin_delta_cmd("thin_delta", thin_delta_main);
-
 //----------------------------------------------------------------

thin-provisioning/thin_dump.cc

@@ -27,39 +27,44 @@
 #include "xml_format.h"
 #include "version.h"
 #include "thin-provisioning/commands.h"
+#include "persistent-data/file_utils.h"
 
+using namespace boost;
 using namespace persistent_data;
 using namespace std;
 using namespace thin_provisioning;
 
-struct flags {
-	bool find_metadata_snap;
-	bool repair;
-};
-
 namespace {
-	int dump_(string const &path, ostream &out, string const &format, struct flags &flags,
-		  block_address metadata_snap) {
+	// FIXME: put the path into the flags
+	struct flags {
+		flags()
+			: repair(false),
+			  use_metadata_snap(false) {
+		}
+
+		bool repair;
+		bool use_metadata_snap;
+		optional<block_address> snap_location;
+	};
+
+	metadata::ptr open_metadata(string const &path, struct flags &flags) {
+		block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY, !flags.use_metadata_snap);
+		metadata::ptr md(flags.use_metadata_snap ? new metadata(bm, flags.snap_location) : new metadata(bm));
+
+		return md;
+	}
+
+	int dump_(string const &path, ostream &out, string const &format, struct flags &flags) {
 		try {
-			metadata::ptr md(new metadata(path, metadata_snap));
+			metadata::ptr md = open_metadata(path, flags);
 			emitter::ptr e;
 
-			if (flags.find_metadata_snap) {
-				uint64_t metadata_snap_root = md->sb_.metadata_snap_; /* FIXME: use thin_pool method? */
-
-				if (metadata_snap_root) {
-					md.reset();
-					md = metadata::ptr(new metadata(path, metadata_snap_root));
-				} else {
-					cerr << "no metadata snapshot found!" << endl;
-					exit(1);
-				}
-			}
-
 			if (format == "xml")
 				e = create_xml_emitter(out);
+
 			else if (format == "human_readable")
 				e = create_human_readable_emitter(out);
+
 			else {
 				cerr << "unknown format '" << format << "'" << endl;
 				exit(1);
@@ -75,13 +80,12 @@ namespace {
 		return 0;
 	}
 
-	int dump(string const &path, char const *output, string const &format, struct flags &flags,
-		 block_address metadata_snap = 0) {
+	int dump(string const &path, char const *output, string const &format, struct flags &flags) {
 		if (output) {
 			ofstream out(output);
-			return dump_(path, out, format, flags, metadata_snap);
+			return dump_(path, out, format, flags);
 		} else
-			return dump_(path, cout, format, flags, metadata_snap);
+			return dump_(path, cout, format, flags);
 	}
 
 	void usage(ostream &out, string const &cmd) {
@@ -96,7 +100,28 @@ namespace {
 	}
 }
 
-int thin_dump_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+thin_dump_cmd::thin_dump_cmd()
+	: command("thin_dump")
+{
+}
+
+void
+thin_dump_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-f|--format} {xml|human_readable}" << endl
+	    << "  {-r|--repair}" << endl
+	    << "  {-m|--metadata-snap} [block#]" << endl
+	    << "  {-o <xml file>}" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+thin_dump_cmd::run(int argc, char **argv)
 {
 	int c;
 	char const *output = NULL;
@@ -105,7 +130,6 @@ int thin_dump_main(int argc, char **argv)
 	string format = "xml";
 	block_address metadata_snap = 0;
 	struct flags flags;
-	flags.find_metadata_snap = flags.repair = false;
 
 	const struct option longopts[] = {
 		{ "help", no_argument, NULL, 'h'},
@@ -120,7 +144,7 @@ int thin_dump_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
 		switch(c) {
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'f':
@@ -132,16 +156,18 @@ int thin_dump_main(int argc, char **argv)
 			break;
 
 		case 'm':
+			flags.use_metadata_snap = true;
 			if (optarg) {
+				// FIXME: deprecate this option
 				metadata_snap = strtoull(optarg, &end_ptr, 10);
 				if (end_ptr == optarg) {
 					cerr << "couldn't parse <metadata_snap>" << endl;
-					usage(cerr, basename(argv[0]));
+					usage(cerr);
 					return 1;
 				}
-			} else
-				flags.find_metadata_snap = true;
 
+				flags.snap_location = metadata_snap;
+			}
 			break;
 
 		case 'o':
@@ -153,20 +179,18 @@ int thin_dump_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
-	return dump(argv[optind], output, format, flags, metadata_snap);
+	return dump(argv[optind], output, format, flags);
 }
 
-base::command thin_provisioning::thin_dump_cmd("thin_dump", thin_dump_main);
-
 //----------------------------------------------------------------

thin-provisioning/thin_ls.cc

@@ -0,0 +1,516 @@
+// Copyright (C) 2015 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include <fstream>
+#include <iostream>
+#include <getopt.h>
+#include <libgen.h>
+
+#include "base/disk_units.h"
+#include "base/grid_layout.h"
+#include "boost/lexical_cast.hpp"
+#include "boost/optional.hpp"
+#include "boost/range.hpp"
+#include "persistent-data/file_utils.h"
+#include "thin-provisioning/commands.h"
+#include "thin-provisioning/human_readable_format.h"
+#include "thin-provisioning/metadata.h"
+#include "thin-provisioning/metadata_dumper.h"
+#include "thin-provisioning/xml_format.h"
+#include "version.h"
+
+using namespace base;
+using namespace boost;
+using namespace persistent_data;
+using namespace std;
+using namespace thin_provisioning;
+
+
+//----------------------------------------------------------------
+
+namespace {
+	class mapping_set {
+	public:
+		mapping_set()
+		: bits_(10240, false) {
+		}
+
+		enum block_state {
+			UNMAPPED,
+			EXCLUSIVE,
+			SHARED
+		};
+
+		void inc(block_address b) {
+			if (get_bit(b * 2))
+				set_bit(b * 2 + 1, true); // shared
+			else
+				set_bit(b * 2, true); // exclusive
+		}
+
+		block_state get_state(block_address b) const {
+			if (get_bit(b * 2)) {
+				if (get_bit(b * 2 + 1))
+					return SHARED;
+				else
+					return EXCLUSIVE;
+			} else
+				return UNMAPPED;
+		}
+
+	private:
+		void ensure_size(block_address bit) const {
+			if (bit >= bits_.size()) {
+				unsigned new_size = bits_.size() * 2;
+				while (new_size < bit)
+					new_size *= 2;
+
+				bits_.resize(new_size, false);
+			}
+		}
+
+		bool get_bit(block_address bit) const {
+			ensure_size(bit);
+			return bits_[bit];
+		}
+
+		void set_bit(block_address bit, bool v) {
+			ensure_size(bit);
+			bits_[bit] = v;
+		}
+
+		mutable vector<bool> bits_;
+	};
+
+	//------------------------------------------------
+
+	enum output_field {
+		DEV_ID,
+		MAPPED_BLOCKS,
+		EXCLUSIVE_BLOCKS,
+		SHARED_BLOCKS,
+
+		MAPPED_SECTORS,
+		EXCLUSIVE_SECTORS,
+		SHARED_SECTORS,
+
+		MAPPED,
+		EXCLUSIVE,
+		SHARED,
+
+		TRANSACTION_ID,
+		CREATION_TIME,
+		SNAPSHOT_TIME	// make sure this is always the last one
+	};
+
+	char const *field_names[] = {
+		"DEV",
+		"MAPPED_BLOCKS",
+		"EXCLUSIVE_BLOCKS",
+		"SHARED_BLOCKS",
+
+		"MAPPED_SECTORS",
+		"EXCLUSIVE_SECTORS",
+		"SHARED_SECTORS",
+
+		"MAPPED",
+		"EXCLUSIVE",
+		"SHARED",
+
+		"TRANSACTION",
+		"CREATE_TIME",
+		"SNAP_TIME"
+	};
+
+	output_field string_to_field(string const &str) {
+		for (unsigned i = 0; i < size(field_names); i++)
+			if (str == field_names[i])
+				return static_cast<output_field>(i);
+
+		throw runtime_error("unknown field");
+		return DEV_ID;
+	}
+
+	string field_to_string(output_field const &f) {
+		return field_names[static_cast<unsigned>(f)];
+	}
+
+	void print_headers(grid_layout &out, vector<output_field> const &fields) {
+		vector<output_field>::const_iterator it;
+		for (it = fields.begin(); it != fields.end(); ++it)
+			out.field(field_to_string(*it));
+
+		out.new_row();
+	}
+
+	//------------------------------------------------
+
+	struct flags {
+		flags()
+			: use_metadata_snap(false),
+			  headers(true) {
+
+			fields.push_back(DEV_ID);
+			fields.push_back(MAPPED);
+			fields.push_back(CREATION_TIME);
+			fields.push_back(SNAPSHOT_TIME);
+		}
+
+		bool use_metadata_snap;
+		bool headers;
+		vector<output_field> fields;
+	};
+
+	//------------------------------------------------
+
+	class mapping_pass1 : public mapping_tree_detail::mapping_visitor {
+	public:
+		mapping_pass1(mapping_set &mappings)
+		: mappings_(mappings) {
+		}
+
+		virtual void visit(btree_path const &path, mapping_tree_detail::block_time const &bt) {
+			mappings_.inc(bt.block_);
+		}
+
+	private:
+		mapping_set &mappings_;
+	};
+
+	class mapping_pass2 : public mapping_tree_detail::mapping_visitor {
+	public:
+		mapping_pass2(mapping_set const &mappings)
+		: mappings_(mappings),
+		  exclusives_(0) {
+		}
+
+		virtual void visit(btree_path const &path, mapping_tree_detail::block_time const &bt) {
+			if (mappings_.get_state(bt.block_) == mapping_set::EXCLUSIVE)
+				exclusives_++;
+		}
+
+		block_address get_exclusives() const {
+			return exclusives_;
+		}
+
+	private:
+		mapping_set const &mappings_;
+		block_address exclusives_;
+	};
+
+	void raise_metadata_damage() {
+		throw std::runtime_error("metadata contains errors (run thin_check for details).");
+	}
+
+	class fatal_mapping_damage : public mapping_tree_detail::damage_visitor {
+	public:
+		virtual void visit(mapping_tree_detail::missing_devices const &d) {
+			raise_metadata_damage();
+		}
+
+		virtual void visit(mapping_tree_detail::missing_mappings const &d) {
+			raise_metadata_damage();
+		}
+	};
+
+	void pass1(metadata::ptr md, mapping_set &mappings, uint64_t dev_id) {
+		dev_tree::key k = {dev_id};
+		optional<uint64_t> dev_root = md->mappings_top_level_->lookup(k);
+
+		if (!dev_root)
+			throw runtime_error("couldn't find mapping tree root");
+
+		single_mapping_tree dev_mappings(*md->tm_, *dev_root,
+				   mapping_tree_detail::block_traits::ref_counter(md->tm_->get_sm()));
+
+		mapping_pass1 pass1(mappings);
+		fatal_mapping_damage dv;
+		walk_mapping_tree(dev_mappings, pass1, dv);
+	}
+
+
+	block_address count_exclusives(metadata::ptr md, mapping_set const &mappings, uint64_t dev_id) {
+		dev_tree::key k = {dev_id};
+		optional<uint64_t> dev_root = md->mappings_top_level_->lookup(k);
+
+		if (!dev_root)
+			throw runtime_error("couldn't find mapping tree root");
+
+		single_mapping_tree dev_mappings(*md->tm_, *dev_root,
+				   mapping_tree_detail::block_traits::ref_counter(md->tm_->get_sm()));
+
+		mapping_pass2 pass2(mappings);
+		fatal_mapping_damage dv;
+		walk_mapping_tree(dev_mappings, pass2, dv);
+		return pass2.get_exclusives();
+	}
+
+	//------------------------------------------------
+
+ 	typedef map<block_address, device_tree_detail::device_details> dd_map;
+
+	class details_extractor : public device_tree_detail::device_visitor {
+	public:
+		void visit(block_address dev_id, device_tree_detail::device_details const &dd) {
+			dd_.insert(make_pair(dev_id, dd));
+		}
+
+		dd_map const &get_details() const {
+			return dd_;
+		}
+
+	private:
+		dd_map dd_;
+	};
+
+	struct fatal_details_damage : public device_tree_detail::damage_visitor {
+		void visit(device_tree_detail::missing_devices const &d) {
+			raise_metadata_damage();
+		}
+	};
+
+	device_tree_detail::damage_visitor::ptr details_damage_policy() {
+		typedef device_tree_detail::damage_visitor::ptr dvp;
+		return dvp(new fatal_details_damage());
+	}
+
+	//------------------------------------------------
+
+	bool pass1_needed(vector<output_field> const &fields) {
+		vector<output_field>::const_iterator it;
+		for (it = fields.begin(); it != fields.end(); ++it) {
+			if (*it == EXCLUSIVE_BLOCKS ||
+			    *it == SHARED_BLOCKS ||
+			    *it == EXCLUSIVE_SECTORS ||
+			    *it == SHARED_SECTORS ||
+			    *it == EXCLUSIVE ||
+			    *it == SHARED)
+				return true;
+		}
+
+		return false;
+	}
+
+	void ls_(string const &path, ostream &out, struct flags &flags) {
+		grid_layout grid;
+
+		block_manager<>::ptr bm(open_bm(path, block_manager<>::READ_ONLY,
+						!flags.use_metadata_snap));
+		metadata::ptr md;
+
+		if (flags.use_metadata_snap)
+			md.reset(new metadata(bm, optional<block_address>()));
+		else
+			md.reset(new metadata(bm));
+
+		block_address block_size = md->sb_.data_block_size_;
+
+		details_extractor de;
+		device_tree_detail::damage_visitor::ptr dd_policy(details_damage_policy());
+		walk_device_tree(*md->details_, de, *dd_policy);
+
+		mapping_set mappings;
+		dd_map::const_iterator it;
+		dd_map const &map = de.get_details();
+
+		bool some_exclusive_fields = pass1_needed(flags.fields);
+		if (some_exclusive_fields) {
+			for (it = map.begin(); it != map.end(); ++it)
+				pass1(md, mappings, it->first);
+		}
+
+		if (flags.headers)
+			print_headers(grid, flags.fields);
+
+		for (it = map.begin(); it != map.end(); ++it) {
+			vector<output_field>::const_iterator f;
+
+			block_address exclusive = 0;
+
+			if (some_exclusive_fields)
+				exclusive = count_exclusives(md, mappings, it->first);
+
+			for (f = flags.fields.begin(); f != flags.fields.end(); ++f) {
+				switch (*f) {
+				case DEV_ID:
+					grid.field(it->first);
+					break;
+
+				case MAPPED_BLOCKS:
+					grid.field(it->second.mapped_blocks_);
+					break;
+
+				case EXCLUSIVE_BLOCKS:
+					grid.field(exclusive);
+					break;
+
+				case SHARED_BLOCKS:
+					grid.field(it->second.mapped_blocks_ - exclusive);
+					break;
+
+				case MAPPED_SECTORS:
+					grid.field(it->second.mapped_blocks_ * block_size);
+					break;
+
+				case EXCLUSIVE_SECTORS:
+					grid.field(exclusive * block_size);
+					break;
+
+				case SHARED_SECTORS:
+					grid.field((it->second.mapped_blocks_ - exclusive) * block_size);
+					break;
+
+
+				case MAPPED:
+					grid.field(
+						format_disk_unit(it->second.mapped_blocks_ * block_size,
+								 UNIT_SECTOR));
+					break;
+
+				case EXCLUSIVE:
+					grid.field(
+						format_disk_unit(exclusive * block_size,
+								 UNIT_SECTOR));
+					break;
+
+				case SHARED:
+					grid.field(
+						format_disk_unit((it->second.mapped_blocks_ - exclusive) *
+								 block_size, UNIT_SECTOR));
+					break;
+
+				case TRANSACTION_ID:
+					grid.field(it->second.transaction_id_);
+					break;
+
+				case CREATION_TIME:
+					grid.field(it->second.creation_time_);
+					break;
+
+				case SNAPSHOT_TIME:
+					grid.field(it->second.snapshotted_time_);
+				}
+			}
+			grid.new_row();
+		}
+
+		grid.render(out);
+	}
+
+	int ls(string const &path, ostream &out, struct flags &flags) {
+		try {
+			ls_(path, out, flags);
+
+		} catch (std::exception &e) {
+			cerr << e.what() << endl;
+			return 1;
+		}
+
+		return 0;
+	}
+}
+
+//----------------------------------------------------------------
+
+thin_ls_cmd::thin_ls_cmd()
+	: command("thin_ls")
+{
+}
+
+void
+thin_ls_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {metadata device}\n"
+	    << "Options:\n"
+	    << "  {-h|--help}\n"
+	    << "  {-m|--metadata-snap}\n"
+	    << "  {--no-headers}\n"
+	    << "  {-o|--format <fields>}\n"
+	    << "  {-V|--version}\n\n"
+	    << "where <fields> is a comma separated list from:\n";
+
+	for (unsigned i = 0; i <= static_cast<unsigned>(SNAPSHOT_TIME); i++)
+            out << "  " << field_to_string(static_cast<output_field>(i)) << "\n";
+}
+
+vector<output_field> parse_fields(string const &str)
+{
+	vector<output_field> fields;
+	stringstream in(str);
+	string item;
+
+	while (getline(in, item, ','))
+		fields.push_back(string_to_field(item));
+
+	return fields;
+}
+
+int
+thin_ls_cmd::run(int argc, char **argv)
+{
+	int c;
+	struct flags flags;
+	const char shortopts[] = "ho:m::V";
+
+	const struct option longopts[] = {
+		{ "help", no_argument, NULL, 'h'},
+		{ "metadata-snap", no_argument, NULL, 'm' },
+		{ "version", no_argument, NULL, 'V'},
+		{ "format", required_argument, NULL, 'o' },
+		{ "no-headers", no_argument, NULL, 1 },
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
+		switch(c) {
+		case 'h':
+			usage(cout);
+			return 0;
+
+		case 'm':
+			flags.use_metadata_snap = true;
+			break;
+
+		case 'o':
+			flags.fields = parse_fields(optarg);
+			break;
+
+		case 'V':
+			cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
+			return 0;
+
+		case 1:
+			flags.headers = false;
+			break;
+
+		default:
+			usage(cerr);
+			return 1;
+		}
+	}
+
+	if (argc == optind) {
+		cerr << "No input file provided." << endl;
+		usage(cerr);
+		return 1;
+	}
+
+	return ls(argv[optind], cout, flags);
+}
+
+//----------------------------------------------------------------

thin-provisioning/thin_metadata_size.cc

@@ -35,6 +35,8 @@
 
 #include <search.h>
 
+using namespace thin_provisioning;
+
 /*----------------------------------------------------------------*/
 
 enum numeric_options { BLOCKSIZE, POOLSIZE, MAXTHINS, NUMERIC, OPT_END};
@@ -324,7 +326,7 @@ static const unsigned mappings_per_block(void)
 static void print_precision(struct global *g, double r, unsigned idx)
 {
 	bool full = g->options.n[NUMERIC] == NO_NUMBER;
-	double rtrunc = truncl(r);
+	double rtrunc = floor(r);
 
 	if (full)
 		printf("%s - ", g->prg);
@@ -362,7 +364,21 @@ static void print_estimated_result(struct global *g)
 	print_precision(g, r, g->options.unit_idx);
 }
 
-int thin_metadata_size_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+thin_metadata_size_cmd::thin_metadata_size_cmd()
+	: command("thin_metadata_size")
+{
+}
+
+void
+thin_metadata_size_cmd::usage(std::ostream &out) const
+{
+	// FIXME: finish
+}
+
+int
+thin_metadata_size_cmd::run(int argc, char **argv)
 {
 	struct global *g = init_prg(*argv);
 
@@ -372,4 +388,4 @@ int thin_metadata_size_main(int argc, char **argv)
 	return 0; /* Doesn't get here... */
 }
 
-base::command thin_provisioning::thin_metadata_size_cmd("thin_metadata_size", thin_metadata_size_main);
+//----------------------------------------------------------------

thin-provisioning/thin_repair.cc

@@ -2,6 +2,7 @@
 #include <getopt.h>
 #include <libgen.h>
 
+#include "persistent-data/file_utils.h"
 #include "thin-provisioning/commands.h"
 #include "human_readable_format.h"
 #include "metadata_dumper.h"
@@ -17,10 +18,12 @@ namespace {
 	int repair(string const &old_path, string const &new_path) {
 		try {
 			// block size gets updated by the restorer
-			metadata::ptr new_md(new metadata(new_path, metadata::CREATE, 128, 0));
+			block_manager<>::ptr new_bm = open_bm(new_path, block_manager<>::READ_WRITE);
+			metadata::ptr new_md(new metadata(new_bm, metadata::CREATE, 128, 0));
 			emitter::ptr e = create_restore_emitter(new_md);
 
-			metadata::ptr old_md(new metadata(old_path));
+			block_manager<>::ptr old_bm = open_bm(old_path, block_manager<>::READ_ONLY);
+			metadata::ptr old_md(new metadata(old_bm));
 			metadata_dump(old_md, e, true);
 
 		} catch (std::exception &e) {
@@ -30,18 +33,28 @@ namespace {
 
 		return 0;
 	}
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-i|--input} <input metadata (binary format)>" << endl
-		    << "  {-o|--output} <output metadata (binary format)>" << endl
-		    << "  {-V|--version}" << endl;
-	}
 }
 
-int thin_repair_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+thin_repair_cmd::thin_repair_cmd()
+	: command("thin_repair")
+{
+}
+
+void
+thin_repair_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-i|--input} <input metadata (binary format)>" << endl
+	    << "  {-o|--output} <output metadata (binary format)>" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+thin_repair_cmd::run(int argc, char **argv)
 {
 	int c;
 	boost::optional<string> input_path, output_path;
@@ -58,7 +71,7 @@ int thin_repair_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
 		switch(c) {
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'i':
@@ -74,26 +87,24 @@ int thin_repair_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (!input_path) {
 		cerr << "no input file provided" << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	if (!output_path) {
 		cerr << "no output file provided" << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 
 	return repair(*input_path, *output_path);
 }
 
-base::command thin_provisioning::thin_repair_cmd("thin_repair", thin_repair_main);
-
 //----------------------------------------------------------------

thin-provisioning/thin_restore.cc

@@ -45,7 +45,8 @@ namespace {
 	int restore(string const &backup_file, string const &dev, bool quiet) {
 		try {
 			// The block size gets updated by the restorer.
-			metadata::ptr md(new metadata(dev, metadata::CREATE, 128, 0));
+			block_manager<>::ptr bm(open_bm(dev, block_manager<>::READ_WRITE));
+			metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0));
 			emitter::ptr restorer = create_restore_emitter(md);
 
 			parse_xml(backup_file, restorer, quiet);
@@ -59,20 +60,32 @@ namespace {
 	}
 
 	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options]" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-i|--input} <input xml file>" << endl
-		    << "  {-o|--output} <output device or file>" << endl
-		    << "  {-q|--quiet}" << endl
-		    << "  {-V|--version}" << endl;
 	}
 }
 
-int thin_restore_main(int argc, char **argv)
+//----------------------------------------------------------------
+
+thin_restore_cmd::thin_restore_cmd()
+	: command("thin_restore")
+{
+}
+
+void
+thin_restore_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options]" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-i|--input} <input xml file>" << endl
+	    << "  {-o|--output} <output device or file>" << endl
+	    << "  {-q|--quiet}" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+thin_restore_cmd::run(int argc, char **argv)
 {
 	int c;
-	char const *prog_name = basename(argv[0]);
 	const char *shortopts = "hi:o:qV";
 	string input, output;
 	bool quiet = false;
@@ -88,7 +101,7 @@ int thin_restore_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
 		switch(c) {
 		case 'h':
-			usage(cout, prog_name);
+			usage(cout);
 			return 0;
 
 		case 'i':
@@ -108,31 +121,29 @@ int thin_restore_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, prog_name);
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc != optind) {
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
         if (input.empty()) {
 		cerr << "No input file provided." << endl << endl;
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
 	if (output.empty()) {
 		cerr << "No output file provided." << endl << endl;
-		usage(cerr, prog_name);
+		usage(cerr);
 		return 1;
 	}
 
 	return restore(input, output, quiet);
 }
 
-base::command thin_provisioning::thin_restore_cmd("thin_restore", thin_restore_main);
-
 //----------------------------------------------------------------

thin-provisioning/thin_rmap.cc

@@ -111,22 +111,30 @@ namespace {
 
 		return region(begin, end);
 	};
-
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}" << endl
-		    << "Options:" << endl
-		    << "  {-h|--help}" << endl
-		    << "  {-V|--version}" << endl
-		    << "  {--region <block range>}*" << endl
-		    << "Where:" << endl
-		    << "  <block range> is of the form <begin>..<one-past-the-end>" << endl
-		    << "  for example 5..45 denotes blocks 5 to 44 inclusive, but not block 45" << endl;
-	}
 }
 
 //----------------------------------------------------------------
 
-int thin_rmap_main(int argc, char **argv)
+thin_rmap_cmd::thin_rmap_cmd()
+	: command("thin_rmap")
+{
+}
+
+void
+thin_rmap_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl
+	    << "  {--region <block range>}*" << endl
+	    << "Where:" << endl
+	    << "  <block range> is of the form <begin>..<one-past-the-end>" << endl
+	    << "  for example 5..45 denotes blocks 5 to 44 inclusive, but not block 45" << endl;
+}
+
+int
+thin_rmap_cmd::run(int argc, char **argv)
 {
 	int c;
 	vector<region> regions;
@@ -141,7 +149,7 @@ int thin_rmap_main(int argc, char **argv)
 	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
 		switch (c) {
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'V':
@@ -161,20 +169,18 @@ int thin_rmap_main(int argc, char **argv)
 			break;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (argc == optind) {
 		cerr << "No input file provided." << endl;
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		exit(1);
 	}
 
 	return rmap(argv[optind], regions);
 }
 
-base::command thin_provisioning::thin_rmap_cmd("thin_rmap", thin_rmap_main);
-
 //----------------------------------------------------------------

thin-provisioning/thin_trim.cc

@@ -6,6 +6,11 @@
 
 #undef BLOCK_SIZE
 
+#ifndef BLKDISCARD
+#define BLKDISCARD _IO(0x12,119)
+#endif
+
+#include "persistent-data/file_utils.h"
 #include "thin-provisioning/commands.h"
 #include "metadata.h"
 #include "version.h"
@@ -118,7 +123,8 @@ namespace {
 	int trim(string const &metadata_dev, string const &data_dev) {
 		// We can trim any block that has zero count in the data
 		// space map.
-		metadata md(metadata_dev, 0);
+		block_manager<>::ptr bm = open_bm(metadata_dev, block_manager<>::READ_ONLY);
+		metadata md(bm);
 
 		if (!md.data_sm_->get_nr_free())
 			return 0;
@@ -133,14 +139,6 @@ namespace {
 		return 0;
 	}
 
-	void usage(ostream &out, string const &cmd) {
-		out << "Usage: " << cmd << " [options] {device|file}\n"
-		    << "Options:\n"
-		    << "  {--pool-inactive}\n"
-		    << "  {-h|--help}\n"
-		    << "  {-V|--version}" << endl;
-	}
-
 	struct flags {
 		boost::optional<string> metadata_dev;
 		boost::optional<string> data_dev;
@@ -149,7 +147,23 @@ namespace {
 
 //----------------------------------------------------------------
 
-int thin_trim_main(int argc, char **argv)
+thin_trim_cmd::thin_trim_cmd()
+	: command("thin_trim")
+{
+}
+
+void
+thin_trim_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " [options] {device|file}\n"
+	    << "Options:\n"
+	    << "  {--pool-inactive}\n"
+	    << "  {-h|--help}\n"
+	    << "  {-V|--version}" << endl;
+}
+
+int
+thin_trim_cmd::run(int argc, char **argv)
 {
 	int c;
 	flags fs;
@@ -175,7 +189,7 @@ int thin_trim_main(int argc, char **argv)
 			break;
 
 		case 'h':
-			usage(cout, basename(argv[0]));
+			usage(cout);
 			return 0;
 
 		case 'V':
@@ -183,13 +197,13 @@ int thin_trim_main(int argc, char **argv)
 			return 0;
 
 		default:
-			usage(cerr, basename(argv[0]));
+			usage(cerr);
 			return 1;
 		}
 	}
 
 	if (!fs.metadata_dev || !fs.data_dev) {
-		usage(cerr, basename(argv[0]));
+		usage(cerr);
 		return 1;
 	}
 

unit-tests/damage_tracker_t.cc

@@ -98,4 +98,15 @@ TEST_F(DamageTrackerTests, gi_bi_gi_bl_gl)
 	assert_damage(dt.good_leaf(15, 20), run64(10ull, 15ull));
 }
 
+TEST_F(DamageTrackerTests, end_resets_tracker)
+{
+	dt.good_internal(0);
+	dt.good_leaf(0, 10);
+	dt.bad_node();
+	assert_damage(dt.end(), run64(10ull));
+
+	assert_no_damage(dt.good_leaf(20, 30));
+	assert_no_damage(dt.end());
+}
+
 //----------------------------------------------------------------