:)

Fix github issue #138

.gitignore

@@ -20,6 +20,7 @@ core
 googletest/
 
 bin/pdata_tools
+bin/pdata_tools_dev
 thin_check
 thin_dump
 thin_restore

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    hooks:
+    -   id: check-merge-conflict
+    -   id: end-of-file-fixer
+    -   id: mixed-line-ending
+    -   id: trailing-whitespace
+-   repo: https://github.com/doublify/pre-commit-rust
+    rev: master
+    hooks:
+    -   id: fmt
+    -   id: clippy

Makefile.in

@@ -20,18 +20,22 @@ V=@
 
 PROGRAMS=\
 	bin/pdata_tools
-
-ifeq ("@TESTING@", "yes")
+DEV_TOOLS=\
+	bin/pdata_tools_dev
 TESTLIBS=\
 	lib/libft.so
-endif
 
-.PHONY: all
-all: $(PROGRAMS) $(TESTLIBS)
+.PHONY: all dev-tools
+all: $(PROGRAMS)
+dev-tools: $(DEV_TOOLS)
+
+ifeq ("@TESTING@", "yes")
+all += $(TESTLIB)
+endif
 
 include contrib/Makefile
 
-SOURCE=\
+COMMON_SOURCE=\
 	base/output_file_requirements.cc \
 	base/application.cc \
 	base/base64.cc \
@@ -50,13 +54,6 @@ SOURCE=\
 	block-cache/copier.cc \
 	block-cache/io_engine.cc \
 	block-cache/mem_pool.cc \
-	caching/cache_check.cc \
-	caching/cache_dump.cc \
-	caching/cache_metadata_size.cc \
-	caching/cache_repair.cc \
-	caching/cache_restore.cc \
-	caching/cache_writeback.cc \
-	caching/commands.cc \
 	caching/hint_array.cc \
 	caching/mapping_array.cc \
 	caching/metadata.cc \
@@ -64,13 +61,8 @@ SOURCE=\
 	caching/restore_emitter.cc \
 	caching/superblock.cc \
 	caching/xml_format.cc \
-	era/commands.cc \
 	era/era_array.cc \
-	era/era_check.cc \
 	era/era_detail.cc \
-	era/era_dump.cc \
-	era/era_invalidate.cc \
-	era/era_restore.cc \
 	era/metadata.cc \
 	era/metadata_dump.cc \
 	era/restore_emitter.cc \
@@ -95,11 +87,7 @@ SOURCE=\
 	persistent-data/space_map.cc \
 	persistent-data/transaction_manager.cc \
 	persistent-data/validators.cc \
-	thin-provisioning/commands.cc \
-	thin-provisioning/cache_stream.cc \
-	thin-provisioning/chunk_stream.cc \
 	thin-provisioning/device_tree.cc \
-	thin-provisioning/fixed_chunk_stream.cc \
 	thin-provisioning/human_readable_format.cc \
 	thin-provisioning/mapping_tree.cc \
 	thin-provisioning/metadata.cc \
@@ -107,40 +95,71 @@ SOURCE=\
 	thin-provisioning/metadata_counter.cc \
 	thin-provisioning/metadata_dumper.cc \
 	thin-provisioning/override_emitter.cc \
-	thin-provisioning/pool_stream.cc \
 	thin-provisioning/restore_emitter.cc \
 	thin-provisioning/rmap_visitor.cc \
 	thin-provisioning/superblock.cc \
+	thin-provisioning/xml_format.cc
+
+TOOLS_SOURCE=\
+	caching/commands.cc \
+	caching/cache_check.cc \
+	caching/cache_dump.cc \
+	caching/cache_metadata_size.cc \
+	caching/cache_repair.cc \
+	caching/cache_restore.cc \
+	caching/cache_writeback.cc \
+	era/commands.cc \
+	era/era_check.cc \
+	era/era_dump.cc \
+	era/era_invalidate.cc \
+	era/era_restore.cc \
+	thin-provisioning/commands.cc \
 	thin-provisioning/thin_check.cc \
 	thin-provisioning/thin_delta.cc \
 	thin-provisioning/thin_dump.cc \
 	thin-provisioning/thin_ls.cc \
 	thin-provisioning/thin_metadata_size.cc \
-	thin-provisioning/thin_pool.cc \
 	thin-provisioning/thin_repair.cc \
 	thin-provisioning/thin_restore.cc \
 	thin-provisioning/thin_rmap.cc \
-	thin-provisioning/thin_trim.cc \
-	thin-provisioning/xml_format.cc
+	thin-provisioning/thin_trim.cc
 
 DEVTOOLS_SOURCE=\
+	caching/cache_debug.cc \
+	caching/devel_commands.cc \
+	dbg-lib/bitset_block_dumper.cc \
+	dbg-lib/command_interpreter.cc \
+	dbg-lib/commands.cc \
+	dbg-lib/index_block_dumper.cc \
+	dbg-lib/output_formatter.cc \
+	dbg-lib/simple_show_traits.cc \
+	dbg-lib/sm_show_traits.cc \
+	era/devel_commands.cc \
+	era/era_debug.cc \
+	thin-provisioning/cache_stream.cc \
+	thin-provisioning/chunk_stream.cc \
 	thin-provisioning/damage_generator.cc \
+	thin-provisioning/devel_commands.cc \
+	thin-provisioning/fixed_chunk_stream.cc \
+	thin-provisioning/pool_stream.cc \
+	thin-provisioning/thin_debug.cc \
+	thin-provisioning/thin_generate_damage.cc \
+	thin-provisioning/thin_generate_mappings.cc \
+	thin-provisioning/thin_generate_metadata.cc \
 	thin-provisioning/thin_journal.cc \
 	thin-provisioning/thin_journal_check.cc \
 	thin-provisioning/thin_ll_dump.cc \
 	thin-provisioning/thin_ll_restore.cc \
-	thin-provisioning/thin_show_duplicates.cc \
-	thin-provisioning/thin_generate_damage.cc \
-	thin-provisioning/thin_generate_metadata.cc \
-	thin-provisioning/thin_generate_mappings.cc \
-	thin-provisioning/variable_chunk_stream.cc \
-	thin-provisioning/thin_show_metadata.cc \
+	thin-provisioning/thin_patch_superblock.cc \
+	thin-provisioning/thin_pool.cc \
 	thin-provisioning/thin_scan.cc \
+	thin-provisioning/thin_show_duplicates.cc \
+	thin-provisioning/thin_show_metadata.cc \
+	thin-provisioning/variable_chunk_stream.cc \
 	ui/ui.cc
 
-ifeq ("@DEVTOOLS@", "yes")
-SOURCE+=$(DEVTOOLS_SOURCE)
-endif
+SOURCE=$(COMMON_SOURCE) $(TOOLS_SOURCE)
+DEV_SOURCE=$(COMMON_SOURCE) $(DEVTOOLS_SOURCE)
 
 ifeq ("@STATIC@", "yes")
 SOURCE += thin-provisioning/static_library_emitter.cc
@@ -153,6 +172,7 @@ CXX:=@CXX@
 AR:=@AR@
 STRIP:=@STRIP@
 OBJECTS:=$(subst .cc,.o,$(SOURCE))
+DEV_OBJECTS:=$(subst .cc,.o,$(DEV_SOURCE))
 
 ifeq ("@STATIC@", "yes")
 EMITTERS += $(PLUGIN_LIBS)
@@ -164,20 +184,14 @@ CFLAGS+=-g -Wall -O3 -fPIC
 CFLAGS+=@LFS_FLAGS@
 CXXFLAGS+=-g -Wall -fPIC -fno-strict-aliasing -std=c++11
 
-ifeq ("@DEVTOOLS@", "yes")
-CXXFLAGS+=-DDEV_TOOLS
-endif
-
 CXXFLAGS+=@CXXOPTIMISE_FLAG@
 CXXFLAGS+=@CXXDEBUG_FLAG@
 CXXFLAGS+=@CXX_STRERROR_FLAG@
 CXXFLAGS+=@LFS_FLAGS@
-INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning
+CPPFLAGS?=@CPPFLAGS@
+CPPFLAGS+=-I$(TOP_BUILDDIR) -I$(TOP_DIR)
 LIBS:=-laio -lexpat -ldl
-
-ifeq ("@DEVTOOLS@", "yes")
-LIBS+=-lncurses
-endif
+DEV_LIBS:=-lncurses
 
 ifeq ("@STATIC_CXX@", "yes")
 CXXLIB+=-Wl,-Bstatic -lstdc++ -Wl,-Bdynamic -Wl,--as-needed
@@ -201,37 +215,29 @@ INSTALL_DIR = $(INSTALL) -m 755 -d
 INSTALL_PROGRAM = $(INSTALL) -m 755
 INSTALL_DATA = $(INSTALL) -p -m 644
 
-ifeq ("@TESTING@", "yes")
-TEST_INCLUDES=\
-	-I$(TOP_DIR) \
-	-Igoogletest/googlemock/include \
-	-Igoogletest/googletest/include
-else
-TEST_INCLUDES=
-endif
-
 .SUFFIXES: .d .txt .8
 
 %.o: %.cc
 	@echo "    [CXX] $<"
 	@mkdir -p $(dir $@)
-	$(V) $(CXX) -c $(INCLUDES) $(CXXFLAGS) -o $@ $<
-	$(V) $(CXX) -MM -MT $(subst .cc,.o,$<) $(INCLUDES) $(TEST_INCLUDES) $(CXXFLAGS) $< > $*.$$$$; \
+	$(V) $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) -o $@ $<
+	$(V) $(CXX) -MM -MT $(subst .cc,.o,$<) $(CPPFLAGS) $(CXXFLAGS) $< > $*.$$$$; \
 	sed 's,\([^ :]*\)\.o[ :]*,\1.o \1.gmo $* : Makefile ,g' < $*.$$$$ > $*.d; \
 	$(RM) $*.$$$$
 
 %.o: %.c
 	@echo "    [CC] $<"
 	@mkdir -p $(dir $@)
-	$(V) $(CC) -c $(INCLUDES) $(CFLAGS) -o $@ $<
-	$(V) $(CC) -MM -MT $(subst .cc,.o,$<) $(INCLUDES) $(TEST_INCLUDES) $(CFLAGS) $< > $*.$$$$; \
+	$(V) $(CC) -c $(CPPFLAGS) $(CFLAGS) -o $@ $<
+	$(V) $(CC) -MM -MT $(subst .cc,.o,$<) $(CPPFLAGS) $(CFLAGS) $< > $*.$$$$; \
 	sed 's,\([^ :]*\)\.o[ :]*,\1.o \1.gmo $* : Makefile ,g' < $*.$$$$ > $*.d; \
 	$(RM) $*.$$$$
 
 %.8: %.txt bin/txt2man
 	@echo "    [txt2man] $<"
 	@mkdir -p $(dir $@)
-	$(V) bin/txt2man -p -t $(basename $(notdir $<)) $< > $@
+	$(V) bin/txt2man -t $(basename $(notdir $<)) \
+	-s 8 -v "System Manager's Manual" -r "Device Mapper Tools" $< > $@
 
 #----------------------------------------------------------------
 
@@ -245,10 +251,16 @@ bin/pdata_tools: $(OBJECTS) $(EMITTERS)
 	@mkdir -p $(dir $@)
 	$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(CXXLIB)
 
+bin/pdata_tools_dev: $(DEV_OBJECTS)
+	@echo "    [LD]  $@"
+	@mkdir -p $(dir $@)
+	$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(DEV_LIBS) $(CXXLIB)
+
 #----------------------------------------------------------------
 
 DEPEND_FILES=\
 	$(subst .cc,.d,$(SOURCE)) \
+	$(subst .cc,.d,$(DEV_SOURCE)) \
 	$(subst .cc,.d,$(TEST_SOURCE)) \
 	$(subst .cc,.d,$(CXX_PROGRAM_SOURCE)) \
 	$(subst .c,.d,$(C_PROGRAM_SOURCE))
@@ -259,7 +271,8 @@ clean:
 	find . -name \*.o -delete
 	find . -name \*.gmo -delete
 	find . -name \*.d -delete
-	$(RM) $(TEST_PROGRAMS) $(PROGRAMS) $(GMOCK_OBJECTS) lib/*.a lib/*.so
+	$(RM) $(PROGRAMS) $(DEV_TOOLS) lib/*.a lib/*.so
+	$(RM) man8/*.8
 
 distclean: clean
 	$(RM) config.cache config.log config.status configure.h version.h Makefile unit-tests/Makefile
@@ -330,17 +343,11 @@ install: bin/pdata_tools $(MANPAGES)
 	$(INSTALL_DATA) man8/era_restore.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/era_invalidate.8 $(MANPATH)/man8
 	$(INSTALL_DATA) man8/thin_trim.8 $(MANPATH)/man8
-ifeq ("@DEVTOOLS@", "yes")
-	ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates
-	ln -s -f pdata_tools $(BINDIR)/thin_ll_dump
-	ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates
-	ln -s -f pdata_tools $(BINDIR)/thin_generate_metadata
-	ln -s -f pdata_tools $(BINDIR)/thin_scan
-endif
 
 .PHONY: install
 
-ifeq ("@TESTING@", "yes")
+#----------------------------------------------------------------
+
 include unit-tests/Makefile
 
 LIBFT_SOURCE=\
@@ -356,11 +363,11 @@ lib/libft.so: $(LIBFT_OBJECTS)
 
 .PHONEY: functional-test unit-test
 
-functional-test: bin/pdata_tools lib/libft.so
+functional-test: $(PROGRAMS) $(DEV_TOOLS) $(TESTLIBS)
 	cd functional-tests && ./run-tests run
 
 test: functional-test unit-test
-endif
+
+#----------------------------------------------------------------
 
 -include $(DEPEND_FILES)
-

base/application.cc

@@ -25,8 +25,14 @@ command::die(string const &msg)
 }
 
 ::uint64_t
-command::parse_uint64(string const &str, string const &desc)
+command::parse_uint64(char const *str, char const *desc)
 {
+	if (!str) {
+		ostringstream out;
+		out << "Couldn't parse " << desc << ": NULL";
+		die(out.str());
+	}
+
 	try {
 		// FIXME: check trailing garbage is handled
 		return lexical_cast<::uint64_t>(str);
@@ -47,7 +53,7 @@ application::run(int argc, char **argv)
 {
 	string cmd = get_basename(argv[0]);
 
-	if (cmd == string("pdata_tools")) {
+	if (cmd.find("pdata_tools") == 0) {
 		argc--;
 		argv++;
 

base/application.h

@@ -19,7 +19,7 @@ namespace base {
 		virtual ~command() {}
 
 		void die(std::string const &msg);
-		uint64_t parse_uint64(std::string const &str, std::string const &desc);
+		uint64_t parse_uint64(char const *str, char const *desc);
 
 
 		virtual void usage(std::ostream &out) const = 0;

base/file_utils.cc

@@ -71,11 +71,18 @@ void
 file_utils::check_file_exists(string const &file, bool must_be_regular_file) {
 	struct stat info;
 	int r = ::stat(file.c_str(), &info);
-	if (r)
-		throw runtime_error("Couldn't stat file");
+	if (r) {
+		ostringstream msg;
+		msg << file << ": " << base::error_string(errno);
+		throw runtime_error(msg.str());
+	}
 
-	if (must_be_regular_file && !S_ISREG(info.st_mode))
-		throw runtime_error("Not a regular file");
+	if (!S_ISREG(info.st_mode)) {
+		if (must_be_regular_file)
+			throw runtime_error("Not a regular file");
+		if (!S_ISBLK(info.st_mode))
+			throw runtime_error("Not a block device or regular file");
+	}
 }
 
 file_utils::file_descriptor
@@ -116,8 +123,11 @@ file_utils::get_file_length(string const &file) {
 	uint64_t nr_bytes;
 
 	int r = ::stat(file.c_str(), &info);
-	if (r)
-		throw runtime_error("Couldn't stat path");
+	if (r) {
+		ostringstream msg;
+		msg << file << ": " << base::error_string(errno);
+		throw runtime_error(msg.str());
+	}
 
 	if (S_ISREG(info.st_mode))
 		// It's okay to cast st_size to a uint64_t value.
@@ -136,9 +146,11 @@ file_utils::get_file_length(string const &file) {
 			throw runtime_error("ioctl BLKGETSIZE64 failed");
 		}
 		::close(fd);
-	} else
-		// FIXME: needs a better message
-		throw runtime_error("bad path");
+	} else {
+		ostringstream msg;
+		msg << file << ": " << "Not a block device or regular file";
+		throw runtime_error(msg.str());
+	}
 
 	return nr_bytes;
 }
@@ -155,8 +167,11 @@ file_utils::zero_superblock(std::string const &path)
         	throw runtime_error("out of memory");
 
 	memset(buffer, 0, SUPERBLOCK_SIZE);
-	if (::write(fd.fd_, buffer, SUPERBLOCK_SIZE) != SUPERBLOCK_SIZE)
+	if (::write(fd.fd_, buffer, SUPERBLOCK_SIZE) != SUPERBLOCK_SIZE) {
+		free(buffer);
 		throw runtime_error("couldn't zero superblock");
+	}
+	free(buffer);
 }
 
 //----------------------------------------------------------------

bin/txt2man

@@ -1,5 +1,4 @@
 #!/bin/sh
-test "$HOME" = ~ || exec ksh $0 "$@"    # try ksh if sh too old (not yet POSIX)
 
 # Copyright (C) 2001, 2002, 2003 Marc Vertes
 
@@ -18,7 +17,7 @@ test "$HOME" = ~ || exec ksh $0 "$@"    # try ksh if sh too old (not yet POSIX)
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
 # 02111-1307, USA.
 
-# txt2man-1.5
+# release 1.7.1
 
 usage()
 {
@@ -27,7 +26,7 @@ NAME
   txt2man - convert flat ASCII text to man page format
 SYNOPSIS
   txt2man [-hpTX] [-t mytitle] [-P pname] [-r rel] [-s sect]
-          [-v vol] [-I txt] [-B txt] [ifile]
+          [-v vol] [-I txt] [-B txt] [-d date] [ifile]
 DESCRIPTION
   txt2man converts the input text into nroff/troff standard man(7)
   macros used to format Unix manual pages. Nice pages can be generated
@@ -45,26 +44,32 @@ DESCRIPTION
   Here is how text patterns are recognized and processed:
   Sections    These headers are defined by a line in upper case, starting
               column 1. If there is one or more leading spaces, a
-	      sub-section will be generated instead.
+              sub-section will be generated instead. Optionally, the
+              Section name can be preceded by a blank line. This is useful
+              for a better visualization of the source text to be used to
+              generate the manpage.
   Paragraphs  They must be separated by a blank line, and left aligned.
+              Alternatively two blank spaces can be used to produce the
+              same result. This option will provide a better visualization
+              of the source text to be used to generate the manpage.
   Tag list    The item definition is separated from the item description
               by at least 2 blank spaces, even before a new line, if
               definition is too long. Definition will be emphasized
               by default.
   Bullet list  
               Bullet list items are defined by the first word being "-"
-	      or "*" or "o".
+              or "*" or "o".
   Enumerated list  
-	      The first word must be a number followed by a dot.
+              The first word must be a number followed by a dot.
   Literal display blocks  
-	      This paragraph type is used to display unmodified text,
-	      for example source code. It must be separated by a blank
-	      line, and be indented. It is primarily used to format
-	      unmodified source code. It will be printed using fixed font
-	      whenever possible (troff).
+              This paragraph type is used to display unmodified text,
+              for example source code. It must be separated by a blank
+              line and be indented by a TAB. It is primarily used to format
+              unmodified source code. It will be printed using fixed font
+              whenever possible (troff).
   Cross references  
-	      A cross reference (another man page) is defined by a word
-	      followed by a number in parenthesis.
+              A cross reference (another man page) is defined by a word
+              followed by a number in parenthesis.
 
   Special sections:
   NAME      The function or command name and short description are set in
@@ -72,37 +77,45 @@ DESCRIPTION
   SYNOPSIS  This section receives a special treatment to identify command
             name, flags and arguments, and propagate corresponding
             attributes later in the text. If a C like function is recognized
-	    (word immediately followed by an open parenthesis), txt2man will
-	    print function name in bold font, types in normal font, and
-	    variables in italic font. The whole section will be printed using
-	    a fixed font family (courier) whenever possible (troff).
+            (word immediately followed by an open parenthesis), txt2man will
+            print function name in bold font, types in normal font, and
+            variables in italic font. The whole section will be printed using
+            a fixed font family (courier) whenever possible (troff).
 
   It is a good practice to embed documentation into source code, by using
-  comments or constant text variables. txt2man allows to do that, keeping
+  comments or constant text variables. txt2man allows one to do that, keeping
   the document source readable, usable even without further formatting
   (i.e. for online help) and easy to write. The result is high quality
   and standard complying document.
 OPTIONS
   -h          The option -h displays help.
+  -d date     Set date in header. Defaults to current date.
   -P pname    Set pname as project name in header. Default to uname -s.
   -p          Probe title, section name and volume.
   -t mytitle  Set mytitle as title of generated man page.
   -r rel      Set rel as project name and release.
-  -s sect     Set sect as section in heading, ususally a value from 1 to 8.
+  -s sect     Set sect as section in heading, usually a value from 1 to 8.
   -v vol      Set vol as volume name, i.e. "Unix user 's manual".
   -I txt      Italicize txt in output. Can be specified more than once.
   -B txt      Emphasize (bold) txt in output. Can be specified more than once.
   -T          Text result previewing using PAGER, usually more(1).
   -X          X11 result previewing using gxditview(1).
 ENVIRONMENT
-  PAGER    name of paging command, usually more(1), or less(1). If not set
-           falls back to more(1).
-EXAMPLE
+  PAGER              name of paging command, usually more(1), or less(1). If not set
+                     falls back to more(1).
+  SOURCE_DATE_EPOCH  Unix timestamp that is used for date in header instead
+                     of current date.
+EXAMPLES
   Try this command to format this text itself:
 
-      $ txt2man -h 2>&1 | txt2man -T
+    $ txt2man -h 2>&1 | txt2man -T
+
+  The following command will generate a manpage level 1 to foo-1.1.0 program,
+  from foo.txt file, used as source code to previously mentioned manpage:
+
+    $ txt2man -d "15 May 2016" -t foo -r foo-1.1.0 -s 1 -v "show stars on screen" foo.txt > foo.1
 HINTS
-  To obtain an overall good formating of output document, keep paragraphs
+  To obtain an overall good formatting of output document, keep paragraphs
   indented correctly. If you have unwanted bold sections, search for
   multiple spaces between words, which are used to identify a tag list
   (term followed by a description). Choose also carefully the name of
@@ -127,35 +140,53 @@ doprobe=
 itxt=
 btxt=
 post=cat
-while getopts :hpTXr:s:t:v:P:I:B: opt
+while getopts :d:hpTXr:s:t:v:P:I:B: opt
 do
 	case $opt in
-	r) rel=$OPTARG;;
-	t) title=$OPTARG;;
-	s) section=$OPTARG;;
-	v) volume=$OPTARG;;
-	P) sys=$OPTARG;;
-	p) doprobe=1;;
-	I) itxt="$OPTARG§$itxt";;
-	B) btxt=$OPTARG;;
-	T) post="groff -mandoc -Tlatin1 | ${PAGER:-more}";;
-	X) post="groff -mandoc -X";;
-	*) usage; exit;;
+	(d) date=$OPTARG;;
+	(r) rel=$OPTARG;;
+	(t) title=$OPTARG;;
+	(s) section=$OPTARG;;
+	(v) volume=$OPTARG;;
+	(P) sys=$OPTARG;;
+	(p) doprobe=1;;
+	(I) itxt="$OPTARG§$itxt";;
+	(B) btxt="$OPTARG§$btxt";;
+	(T) post="groff -mandoc -Tlatin1 | ${PAGER:-more}";;
+	(X) post="groff -mandoc -TX100-12 -rS12";;
+	(*) usage; exit;;
 	esac
 done
 shift $(($OPTIND - 1))
 
+# Compatibility wrapper for BSD/GNU date, for parsing dates
+if date -j >/dev/null 2>&1; then
+  pdate() { date -u -j -f '@%s' "$@"; }
+else
+  pdate() { date -u -d "$@"; }
+fi
+
+if [ -n "$SOURCE_DATE_EPOCH" ]; then
+  date=$(LC_ALL=C pdate "@$SOURCE_DATE_EPOCH" +'%d %B %Y')
+fi
+date=${date:-$(LC_ALL=C date -u +'%d %B %Y')}
+
 if test "$doprobe"
 then
 	title=${1##*/}; title=${title%.txt}
-	section="8"
-	volume="System Manager's Manual"
+	if grep -q '#include ' $1
+	then
+		section=${section:-3}
+		volume=${volume:-"$sys Programmer's Manual"}
+	else
+		section=${section:-1}
+		volume=${volume:-"$sys Reference Manual"}
+	fi
 	# get release from path
-	#rel=$(pwd | sed 's:/.*[^0-9]/::g; s:/.*::g')
-	rel="Device Mapper Tools"
+	rel=${rel:-"$(pwd | sed 's:/.*[^0-9]/::g; s:/.*::g')"}
 fi
 
-head=".\\\" Text automatically generated by txt2man
+head="\" Text automatically generated by txt2man
 .TH $title $section \"$rel\" \"$volume\""
 
 # All tabs converted to spaces
@@ -163,11 +194,11 @@ expand $* |
 # gawk is needed because use of non standard regexp
 gawk --re-interval -v head="$head" -v itxt="$itxt" -v btxt="$btxt" '
 BEGIN {
-	print head
+	print ".\\" head
 	avar[1] = btxt; avar[2] = itxt
 	for (k in avar) {
 		mark = (k == 1) ? "\\fB" : "\\fI"
-		split(avar[k], tt, "§")
+		split(avar[k], tt, "§")
 		for (i in tt)
 			if (tt[i] != "")
 				subwords["\\<" tt[i] "\\>"] = mark tt[i] "\\fP"
@@ -179,11 +210,11 @@ BEGIN {
 }
 {
 	# to avoid some side effects in regexp
-	sub(/\.\.\./, "\\.\\.\\.")
+	gsub(/\.\.\./, "\\.\\.\\.")
 	# remove spaces in empty lines
 	sub(/^ +$/,"")
 }
-/^[[:upper:][:space:]]+$/ {
+/^[:space:]*[[:upper:][:digit:]]+[[:upper:][:space:][:digit:][:punct:]]+$/ {
 	# Section header
 	if ((in_bd + 0) == 1) {
 		in_bd = 0
@@ -199,8 +230,10 @@ BEGIN {
 		print ".SS" $0
 	sub(/^ +/, "")
 	section = $0
-	if (section == "SYNOPSIS")
+	if (section == "SYNOPSIS") {
 		print ".nf\n.fam C"
+		in_bd = 1
+	}
 	ls = 0		# line start index
 	pls = 0		# previous line start index
 	pnzls = 0	# previous non zero line start index
@@ -216,7 +249,7 @@ BEGIN {
 		pnzls = ls
 	match($0, /[^ ]/)
 	ls = RSTART
-	if (pls == 0 && pnzls > 0 && ls > pnzls && $1 !~ /^[0-9\-\*\o]\.*$/) {
+	if (in_bd == 0 && pls == 0 && pnzls > 0 && ls > pnzls && $1 !~ /^[\-\*o]$|^[0-9]+\.$/) {
 		# example display block
 		if (prevblankline == 1) {
 			print ".PP"
@@ -230,8 +263,10 @@ BEGIN {
 		ind[0] = ls
 }
 (in_bd + 0) == 1 {
-	# In example display block
-	if (ls != 0 && ls < eoff) {
+	# In block display
+	if (section == "SYNOPSIS")
+		;
+	else if (ls != 0 && ls < eoff) {
 		# End of litteral display block
 		in_bd = 0
 		print ".fam T\n.fi"
@@ -244,11 +279,12 @@ section == "NAME" {
 section == "SYNOPSIS" {
 	# Identify arguments of fcts and cmds
 	if (type["SYNOPSIS"] == "") {
-		if (index($0, "(") == 0 && index($0, ")") == 0 &&
-		    index($0, "#include") == 0)
-			type["SYNOPSIS"] = "cmd"
-		else
+		if ($0 ~ /\(/)
 			type["SYNOPSIS"] = "fct"
+		else if ($1 == "struct" || $2 == "struct")
+			type["SYNOPSIS"] = "struct"
+		else if ($1 && $1 !~ /^#|typedef|struct|union|enum/)
+			type["SYNOPSIS"] = "cmd"
 	}
 	if (type["SYNOPSIS"] == "cmd") {
 		# Line is a command line
@@ -263,19 +299,19 @@ section == "SYNOPSIS" {
 			if (a ~ /^[^\-]/)
 				subwords["\\<" a "\\>"] = "\\fI" a "\\fP"
 		}
-	} else {
+	} else if (type["SYNOPSIS"] == "fct") {
 		# Line is a C function definition
-		if ($1 == "typedef")
-			subwords["\\<" $2 "\\>"] = "\\fI" $2 "\\fP"
-		else if ($1 == "#define")
+		if ($1 == "typedef") {
+			if ($0 !~ /\(\*/)
+				subwords["\\<" $2 "\\>"] = "\\fI" $2 "\\fP"
+		} else if ($1 == "#define")
 			subwords["\\<" $2 "\\>"] = "\\fI" $2 "\\fP"
 		for (i = 1; i <= NF; i++) {
-			if ($i ~ /[\,\)]/) {
+			if ($i ~ /[,\)];*$/) {
 				a = $i
 				sub(/.*\(/, "", a)
 				gsub(/\W/, "", a)
-				if (a !~ /^void$/)
-					subwords["\\<" a "\\>"] = "\\fI" a "\\fP"
+				subwords["\\<" a "\\>"] = "\\fI" a "\\fP"
 			}
 		}
 	}
@@ -298,13 +334,19 @@ section == "SYNOPSIS" {
 		}
 	}
 	# word attributes
-	for (i in subwords)
-		gsub(i, subwords[i])
+	n = asorti(subwords, indices)
+	for (i = 1; i <= n; i++)
+		gsub(indices[i], subwords[indices[i]])
 	# shell options
 	gsub(/\B\-+\w+(\-\w+)*/, "\\fB&\\fP")
 	# unprotect dots inside words
 	gsub(/_dOt_/, ".")
 
+	if (section == "SYNOPSIS") {
+		sub(/^  /, "")
+		print
+		next
+	}
 	if (match($0, /[^ ]  +/) > 0) {
 		# tag list item
 		adjust_indent()
@@ -347,6 +389,8 @@ section == "SYNOPSIS" {
 	}
 	if (section != "SYNOPSIS" || $0 ~ /^ {1,4}/)
 		sub(/ */,"")
+	# Protect lines starting by simple quotes
+	sub(/^'\''/, "\\(cq")
 	print
 }
 

block-cache/copier.cc

@@ -10,12 +10,14 @@ using namespace std;
 
 copier::copier(io_engine &engine,
 	       string const &src, string const &dest,
-	       sector_t block_size, size_t mem)
+	       sector_t block_size, size_t mem,
+	       sector_t src_offset, sector_t dest_offset)
 	: pool_(block_size * 512, mem, PAGE_SIZE),
 	  block_size_(block_size),
 	  engine_(engine),
 	  src_handle_(engine_.open_file(src, io_engine::M_READ_ONLY)),
 	  dest_handle_(engine_.open_file(dest, io_engine::M_READ_WRITE)),
+	  src_offset_(src_offset), dest_offset_(dest_offset),
 	  genkey_count_(0)
 {
 }
@@ -45,8 +47,8 @@ copier::issue(copy_op const &op)
 
 	auto r = engine_.issue_io(src_handle_,
 				  io_engine::D_READ,
-				  to_sector(op.src_b),
-				  to_sector(op.src_e),
+				  to_src_sector(op.src_b),
+				  to_src_sector(op.src_e),
 				  data,
 				  key);
 
@@ -151,8 +153,8 @@ copier::wait_successful(io_engine::wait_result const &p)
 		j.op.read_complete = true;
 		if (!engine_.issue_io(dest_handle_,
 				      io_engine::D_WRITE,
-				      to_sector(j.op.dest_b),
-				      to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)),
+				      to_dest_sector(j.op.dest_b),
+				      to_dest_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)),
 				      j.data,
 				      it->first)) {
 			complete(j);
@@ -177,9 +179,15 @@ copier::complete(copy_job const &j)
 }
 
 sector_t
-copier::to_sector(block_address b) const
+copier::to_src_sector(block_address b) const
 {
-	return b * block_size_;
+	return src_offset_ + b * block_size_;
+}
+
+sector_t
+copier::to_dest_sector(block_address b) const
+{
+	return dest_offset_ + b * block_size_;
 }
 
 unsigned

block-cache/copier.h

@@ -61,13 +61,22 @@ namespace bcache {
 	public:
 		copier(io_engine &engine,
 		       std::string const &src, std::string const &dest,
-		       sector_t block_size, size_t mem);
+		       sector_t block_size, size_t mem,
+		       sector_t src_offset, sector_t dest_offset);
 		~copier();
 
 		sector_t get_block_size() const {
 			return block_size_;
 		}
 
+		sector_t get_src_offset() const {
+			return src_offset_;
+		}
+
+		sector_t get_dest_offset() const {
+			return dest_offset_;
+		}
+
 		// Blocks if out of memory.
 		void issue(copy_op const &op);
 
@@ -83,7 +92,8 @@ namespace bcache {
 		void wait_();
 		void complete(copy_job const &j);
 
-		sector_t to_sector(block_address b) const;
+		sector_t to_src_sector(block_address b) const;
+		sector_t to_dest_sector(block_address b) const;
 		unsigned genkey();
 
 		mempool pool_;
@@ -91,6 +101,8 @@ namespace bcache {
 		io_engine &engine_;
 		io_engine::handle src_handle_;
 		io_engine::handle dest_handle_;
+		sector_t src_offset_;
+		sector_t dest_offset_;
 		unsigned genkey_count_;
 
 		using job_map = std::map<unsigned, copy_job>;

block-cache/io_engine.cc

@@ -174,9 +174,6 @@ aio_engine::wait_(timespec *ts)
 		cbs_.free(cb);
 		return optional<wait_result>(make_pair(false, context));
 	}
-
-	// shouldn't get here
-	return optional<wait_result>(make_pair(false, 0));
 }
 
 struct timespec

caching/cache_debug.cc

@@ -0,0 +1,231 @@
+// Copyright (C) 2012 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include <boost/lexical_cast.hpp>
+#include <getopt.h>
+#include <iostream>
+#include <string>
+
+#include "dbg-lib/array_block_dumper.h"
+#include "dbg-lib/btree_node_dumper.h"
+#include "dbg-lib/bitset_block_dumper.h"
+#include "dbg-lib/command_interpreter.h"
+#include "dbg-lib/commands.h"
+#include "dbg-lib/index_block_dumper.h"
+#include "dbg-lib/output_formatter.h"
+#include "dbg-lib/sm_show_traits.h"
+#include "persistent-data/file_utils.h"
+#include "persistent-data/space-maps/disk_structures.h"
+#include "caching/commands.h"
+#include "caching/metadata.h"
+#include "version.h"
+
+using namespace dbg;
+using namespace persistent_data;
+using namespace std;
+using namespace caching;
+
+//----------------------------------------------------------------
+
+namespace {
+	class help : public dbg::command {
+		virtual void exec(strings const &args, ostream &out) {
+			out << "Commands:" << endl
+			    << "  superblock [block#]" << endl
+			    << "  block_node <block# of array block-tree node>" << endl
+			    << "  bitset_block <block# of bitset block>" << endl
+			    << "  index_block <block# of metadata space map root>" << endl
+			    << "  mapping_block <block# of mappings array block>" << endl
+			    << "  exit" << endl;
+		}
+	};
+
+	class show_superblock : public dbg::command {
+	public:
+		explicit show_superblock(block_manager::ptr bm)
+			: bm_(bm) {
+		}
+
+		virtual void exec(strings const &args, ostream &out) {
+			if (args.size() > 2)
+				throw runtime_error("incorrect number of arguments");
+
+			block_address b = caching::SUPERBLOCK_LOCATION;
+			if (args.size() == 2)
+				b = boost::lexical_cast<block_address>(args[1]);
+			caching::superblock sb = read_superblock(bm_, b);
+
+			formatter::ptr f = create_xml_formatter();
+			ostringstream version;
+			field(*f, "csum", sb.csum);
+			field(*f, "flags", sb.flags.encode());
+			field(*f, "blocknr", sb.blocknr);
+			field(*f, "uuid", sb.uuid); // FIXME: delimit, and handle non-printable chars
+			field(*f, "magic", sb.magic);
+			field(*f, "version", sb.version);
+			field(*f, "policy_name", reinterpret_cast<char const*>(sb.policy_name));
+			version << sb.policy_version[0] << "."
+				<< sb.policy_version[1] << "."
+				<< sb.policy_version[2];
+			field(*f, "policy_version", version.str().c_str());
+			field(*f, "policy_hint_size", sb.policy_hint_size);
+
+			sm_disk_detail::sm_root_disk const *d;
+			sm_disk_detail::sm_root v;
+			{
+				d = reinterpret_cast<sm_disk_detail::sm_root_disk const *>(sb.metadata_space_map_root);
+				sm_disk_detail::sm_root_traits::unpack(*d, v);
+				formatter::ptr f2 = create_xml_formatter();
+				sm_root_show_traits::show(f2, "value", v);
+				f->child("metadata_space_map_root", f2);
+			}
+
+			field(*f, "mapping_root", sb.mapping_root);
+			if (sb.version >= 2)
+				field(*f, "dirty_root", *sb.dirty_root);
+			field(*f, "hint_root", sb.hint_root);
+			field(*f, "discard_root", sb.discard_root);
+			field(*f, "discard_block_size", sb.discard_block_size);
+			field(*f, "discard_nr_blocks", sb.discard_nr_blocks);
+			field(*f, "data_block_size", sb.data_block_size);
+			field(*f, "metadata_block_size", sb.metadata_block_size);
+			field(*f, "cache_blocks", sb.cache_blocks);
+			field(*f, "compat_flags", sb.compat_flags);
+			field(*f, "compat_ro_flags", sb.compat_ro_flags);
+			field(*f, "incompat_flags", sb.incompat_flags);
+			field(*f, "read_hits", sb.read_hits);
+			field(*f, "read_misses", sb.read_misses);
+			field(*f, "write_hits", sb.write_hits);
+			field(*f, "write_misses", sb.write_misses);
+
+			f->output(out, 0);
+		}
+
+	private:
+		block_manager::ptr bm_;
+	};
+
+	class mapping_show_traits : public caching::mapping_traits {
+	public:
+		typedef mapping_traits value_trait;
+
+		static void show(formatter::ptr f, string const &key, caching::mapping const &value) {
+			field(*f, "oblock", value.oblock_);
+			field(*f, "flags", value.flags_);
+		}
+	};
+
+	//--------------------------------
+
+	template <typename ShowTraits>
+	dbg::command::ptr
+	create_btree_node_handler(block_manager::ptr bm) {
+		return create_block_handler(bm, create_btree_node_dumper<ShowTraits>());
+	}
+
+	template <typename ShowTraits>
+	dbg::command::ptr
+	create_array_block_handler(block_manager::ptr bm,
+				   typename ShowTraits::value_trait::ref_counter rc) {
+		return create_block_handler(bm, create_array_block_dumper<ShowTraits>(rc));
+	}
+
+	dbg::command::ptr
+	create_bitset_block_handler(block_manager::ptr bm) {
+		return create_block_handler(bm, create_bitset_block_dumper());
+	}
+
+	dbg::command::ptr
+	create_index_block_handler(block_manager::ptr bm) {
+		return create_block_handler(bm, create_index_block_dumper());
+	}
+
+	int debug(string const &path) {
+		using dbg::command;
+
+		try {
+			block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY);
+			command_interpreter::ptr interp = create_command_interpreter(cin, cout);
+			interp->register_command("hello", create_hello_handler());
+			interp->register_command("superblock", command::ptr(new show_superblock(bm)));
+			interp->register_command("block_node", create_btree_node_handler<uint64_show_traits>(bm));
+			interp->register_command("bitset_block", create_bitset_block_handler(bm));
+			interp->register_command("index_block", create_index_block_handler(bm));
+			interp->register_command("mapping_block", create_array_block_handler<mapping_show_traits>(bm,
+					mapping_traits::ref_counter()));
+			interp->register_command("help", command::ptr(new help));
+			interp->register_command("exit", create_exit_handler(interp));
+			interp->enter_main_loop();
+
+		} catch (std::exception &e) {
+			cerr << e.what();
+			return 1;
+		}
+
+		return 0;
+	}
+}
+
+//----------------------------------------------------------------
+
+cache_debug_cmd::cache_debug_cmd()
+	: command("cache_debug")
+{
+}
+
+void
+cache_debug_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+cache_debug_cmd::run(int argc, char **argv)
+{
+	int c;
+	const char shortopts[] = "hV";
+	const struct option longopts[] = {
+		{ "help", no_argument, NULL, 'h'},
+		{ "version", no_argument, NULL, 'V'},
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
+		switch(c) {
+		case 'h':
+			usage(cout);
+			return 0;
+
+		case 'V':
+			cerr << THIN_PROVISIONING_TOOLS_VERSION << endl;
+			return 0;
+		}
+	}
+
+	if (argc == optind) {
+		usage(cerr);
+		exit(1);
+	}
+
+	return debug(argv[optind]);
+}
+
+//----------------------------------------------------------------

caching/cache_repair.cc

@@ -32,7 +32,7 @@ namespace {
 	int repair(string const &old_path, string const &new_path) {
 		bool metadata_touched = false;
 		try {
-			file_utils::check_file_exists(new_path, false);
+			file_utils::check_file_exists(old_path, false);
 			metadata_touched = true;
 			metadata_dump(open_metadata_for_read(old_path),
 				      output_emitter(new_path),

caching/cache_writeback.cc

@@ -31,6 +31,8 @@ namespace {
 		flags()
 			: cache_size(4 * 1024 * 1024),
 			  sort_buffers(16 * 1024),
+			  origin_dev_offset(0),
+			  fast_dev_offset(0),
 			  list_failed_blocks(false),
 			  update_metadata(true) {
 		}
@@ -52,6 +54,8 @@ namespace {
 		maybe_string metadata_dev;
 		maybe_string origin_dev;
 		maybe_string fast_dev;
+		sector_t origin_dev_offset;
+		sector_t fast_dev_offset;
 		bool list_failed_blocks;
 		bool update_metadata;
 	};
@@ -297,13 +301,15 @@ namespace {
 
 	int writeback_(flags const &f) {
 		block_manager::ptr bm = open_bm(*f.metadata_dev, block_manager::READ_WRITE);
-		metadata md(bm);
+		metadata md(bm, true);
 
 		// FIXME: we're going to have to copy runs to get the through put with small block sizes
 		unsigned max_ios = f.cache_size / (md.sb_.data_block_size << SECTOR_SHIFT);
 		aio_engine engine(max_ios);
 		copier c(engine, *f.fast_dev, *f.origin_dev,
-			 md.sb_.data_block_size, f.cache_size);
+			 md.sb_.data_block_size, f.cache_size,
+			 f.fast_dev_offset >> SECTOR_SHIFT,
+			 f.origin_dev_offset >> SECTOR_SHIFT);
 
 		auto bar = create_progress_bar("Copying data");
 		copy_visitor cv(c, f.sort_buffers, clean_shutdown(md), f.list_failed_blocks,
@@ -364,6 +370,8 @@ cache_writeback_cmd::usage(std::ostream &out) const
 	    << "\t\t--buffer-size-meg <size>\n"
 	    << "\t\t--list-failed-blocks\n"
 	    << "\t\t--no-metadata-update\n"
+	    << "\t\t--origin-device-offset <bytes>\n"
+	    << "\t\t--fast-device-offset <bytes>\n"
 	    << "Options:\n"
 	    << "  {-h|--help}\n"
 	    << "  {-V|--version}" << endl;
@@ -382,6 +390,8 @@ cache_writeback_cmd::run(int argc, char **argv)
 		{ "buffer-size-meg", required_argument, NULL, 3 },
 		{ "list-failed-blocks", no_argument, NULL, 4 },
 		{ "no-metadata-update", no_argument, NULL, 5 },
+		{ "origin-device-offset", required_argument, NULL, 6 },
+		{ "fast-device-offset", required_argument, NULL, 7 },
 		{ "help", no_argument, NULL, 'h'},
 		{ "version", no_argument, NULL, 'V'},
 		{ NULL, no_argument, NULL, 0 }
@@ -413,6 +423,14 @@ cache_writeback_cmd::run(int argc, char **argv)
 			fs.update_metadata = false;
 			break;
 
+		case 6:
+			fs.origin_dev_offset = parse_uint64(optarg, "origin dev offset");
+			break;
+
+		case 7:
+			fs.fast_dev_offset = parse_uint64(optarg, "fast dev offset");
+			break;
+
 		case 'h':
 			usage(cout);
 			return 0;
@@ -452,6 +470,13 @@ cache_writeback_cmd::run(int argc, char **argv)
 		return 1;
 	}
 
+	if (fs.origin_dev_offset & (SECTOR_SHIFT - 1) ||
+	    fs.fast_dev_offset & (SECTOR_SHIFT - 1)) {
+		cerr << "Offset must be sector-aligned\n\n";
+		usage(cerr);
+		return 1;
+	}
+
 	return writeback(fs);
 }
 

caching/commands.h

@@ -70,6 +70,18 @@ namespace caching {
 		virtual int run(int argc, char **argv);
 	};
 
+	//------------------------------------------------------
+
+	class cache_debug_cmd : public base::command {
+	public:
+		cache_debug_cmd();
+
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	//------------------------------------------------------
+
 	void register_cache_commands(base::application &app);
 }
 

caching/devel_commands.cc

@@ -0,0 +1,14 @@
+#include "caching/commands.h"
+
+using namespace base;
+using namespace caching;
+
+//----------------------------------------------------------------
+
+void
+caching::register_cache_commands(application &app)
+{
+	app.add_cmd(command::ptr(new cache_debug_cmd));
+}
+
+//----------------------------------------------------------------

caching/hint_array.cc

@@ -56,9 +56,6 @@ namespace {
 		default:
 			throw runtime_error("invalid hint width");
 		}
-
-		// never get here
-		return std::shared_ptr<array_base>();
 	}
 
 	//--------------------------------
@@ -93,9 +90,6 @@ namespace {
 		default:
 			throw runtime_error("invalid hint width");
 		}
-
-		// never get here
-		return std::shared_ptr<array_base>();
 	}
 
 	//--------------------------------

caching/metadata.cc

@@ -45,9 +45,9 @@ metadata::metadata(block_manager::ptr bm, open_type ot, unsigned metadata_versio
 	}
 }
 
-metadata::metadata(block_manager::ptr bm)
+metadata::metadata(block_manager::ptr bm, bool read_space_map)
 {
-	open_metadata(bm);
+	open_metadata(bm, read_space_map);
 }
 
 void
@@ -90,11 +90,16 @@ metadata::create_metadata(block_manager::ptr bm, unsigned metadata_version)
 }
 
 void
-metadata::open_metadata(block_manager::ptr bm)
+metadata::open_metadata(block_manager::ptr bm, bool read_space_map)
 {
 	tm_ = open_tm(bm);
 	sb_ = read_superblock(tm_->get_bm());
 
+	if (read_space_map) {
+		metadata_sm_ = open_metadata_sm(*tm_, &sb_.metadata_space_map_root);
+		tm_->set_sm(metadata_sm_);
+	}
+
 	mappings_ = mapping_array::ptr(
 		new mapping_array(*tm_,
 				  mapping_array::ref_counter(),

caching/metadata.h

@@ -27,7 +27,7 @@ namespace caching {
 		typedef std::shared_ptr<metadata> ptr;
 
 		metadata(block_manager::ptr bm, open_type ot, unsigned metadata_version = 2); // Create only
-		metadata(block_manager::ptr bm);
+		metadata(block_manager::ptr bm, bool read_space_map = false);
 
 		void commit(bool clean_shutdown = true);
 		void setup_hint_array(size_t width);
@@ -46,7 +46,7 @@ namespace caching {
 		void init_superblock();
 
 		void create_metadata(block_manager::ptr bm, unsigned metadata_version);
-		void open_metadata(block_manager::ptr bm);
+		void open_metadata(block_manager::ptr bm, bool read_space_map);
 
 		void commit_space_map();
 		void commit_mappings();

configure.ac

@@ -150,14 +150,6 @@ AC_ARG_ENABLE(testing,
               TESTING=$enableval, TESTING=no)
 AC_MSG_RESULT($TESTING)
 
-################################################################################
-dnl -- Enable development tools
-AC_MSG_CHECKING(whether to enable development tools)
-AC_ARG_ENABLE(dev-tools,
-	      AC_HELP_STRING(--enable-dev-tools, [enable development tools in the makefile]),
-	      DEVTOOLS=$enableval, DEVTOOLS=no)
-AC_MSG_RESULT($DEVTOOLS)
-
 ################################################################################
 dnl -- Enable static libstdc++
 AC_MSG_CHECKING(whether to statically link libstdc++)
@@ -214,6 +206,5 @@ Makefile
 contrib/Makefile
 unit-tests/Makefile
 version.h
-src/version.rs
 ])
 AC_OUTPUT

contrib/Makefile.in

@@ -18,7 +18,7 @@ contrib/%.a: contrib/%.o
 	$(V)echo "    [AR] $@"
 	$(V)$(AR) rcs $@ $^
 
-contrib/%.so: contrib/%.a
+contrib/%.so: contrib/%.o
 	$(V)echo "    [LD] $@"
 	$(V)$(CC) -shared -Wl,-soname,$@ -o $@ $<
 

dbg-lib/array_block_dumper.h

@@ -0,0 +1,53 @@
+#include "dbg-lib/block_dumper.h"
+#include "dbg-lib/output_formatter.h"
+#include "persistent-data/data-structures/array_block.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	using persistent_data::block_manager;
+	using persistent_data::array_block;
+
+	template <typename ShowTraits>
+	class array_block_dumper : public block_dumper {
+	public:
+		array_block_dumper(typename ShowTraits::value_trait::ref_counter rc)
+			: rc_(rc) {
+		}
+
+		virtual void show(block_manager::read_ref &rr, std::ostream &out) {
+			rblock b(rr, rc_);
+			show_array_entries(b, out);
+		}
+
+	private:
+		typedef array_block<typename ShowTraits::value_trait, block_manager::read_ref> rblock;
+
+		void show_array_entries(rblock const& b, std::ostream &out) {
+			formatter::ptr f = create_xml_formatter();
+			uint32_t nr_entries = b.nr_entries();
+
+			field(*f, "max_entries", b.max_entries());
+			field(*f, "nr_entries", nr_entries);
+			field(*f, "value_size", b.value_size());
+
+			for (unsigned i = 0; i < nr_entries; i++) {
+				formatter::ptr f2 = create_xml_formatter();
+				ShowTraits::show(f2, "value", b.get(i));
+				f->child(boost::lexical_cast<std::string>(i), f2);
+			}
+
+			f->output(out, 0);
+		}
+
+		typename ShowTraits::value_trait::ref_counter rc_;
+	};
+
+	template <typename ShowTraits>
+	block_dumper::ptr
+	create_array_block_dumper(typename ShowTraits::value_trait::ref_counter rc) {
+		return block_dumper::ptr(new array_block_dumper<ShowTraits>(rc));
+	}
+}
+
+//----------------------------------------------------------------

dbg-lib/bitset_block_dumper.cc

@@ -0,0 +1,137 @@
+#include "dbg-lib/bitset_block_dumper.h"
+#include "dbg-lib/output_formatter.h"
+#include "persistent-data/data-structures/array_block.h"
+#include "persistent-data/data-structures/simple_traits.h"
+
+using namespace dbg;
+using namespace persistent_data;
+
+//----------------------------------------------------------------
+
+namespace {
+	class bitset_block_dumper : public dbg::block_dumper {
+		typedef array_block<uint64_traits, block_manager::read_ref> rblock;
+	public:
+		explicit bitset_block_dumper()
+			: BITS_PER_ARRAY_ENTRY(64) {
+		}
+
+		virtual void show(block_manager::read_ref &rr, ostream &out) {
+			rblock b(rr, rc_);
+			show_bitset_entries(b, out);
+		}
+
+	private:
+		void show_bitset_entries(rblock const& b, ostream &out) {
+			formatter::ptr f = create_xml_formatter();
+			uint32_t nr_entries = b.nr_entries();
+
+			field(*f, "max_entries", b.max_entries());
+			field(*f, "nr_entries", nr_entries);
+			field(*f, "value_size", b.value_size());
+
+			uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY;
+			std::pair<uint32_t, uint32_t> range = next_set_bits(b, 0);
+			for (; range.first < end_pos; range = next_set_bits(b, range.second)) {
+				formatter::ptr f2 = create_xml_formatter();
+				field(*f2, "begin", range.first);
+				field(*f2, "end", range.second);
+				f->child("set_bits", f2);
+			}
+
+			f->output(out, 0);
+		}
+
+		// Returns the range of set bits, starts from the offset.
+		pair<uint32_t, uint32_t> next_set_bits(rblock const &b, uint32_t offset) {
+			uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY;
+			uint32_t begin = find_first_set(b, offset);
+
+			if (begin == end_pos) // not found
+				return make_pair(end_pos, end_pos);
+
+			uint32_t end = find_first_unset(b, begin + 1);
+			return make_pair(begin, end);
+		}
+
+		// Returns the position (zero-based) of the first bit set
+		// in the array block, starts from the offset.
+		// Returns the pass-the-end position if not found.
+		uint32_t find_first_set(rblock const &b, uint32_t offset) {
+			uint32_t entry = offset / BITS_PER_ARRAY_ENTRY;
+			uint32_t nr_entries = b.nr_entries();
+
+			if (entry >= nr_entries)
+				return entry * BITS_PER_ARRAY_ENTRY;
+
+			uint32_t idx = offset % BITS_PER_ARRAY_ENTRY;
+			uint64_t v = b.get(entry++) >> idx;
+			while (!v && entry < nr_entries) {
+				v = b.get(entry++);
+				idx = 0;
+			}
+
+			if (!v) // not found
+				return entry * BITS_PER_ARRAY_ENTRY;
+
+			return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + ffsll(static_cast<long long>(v)) - 1;
+		}
+
+		// Returns the position (zero-based) of the first zero bit
+		// in the array block, starts from the offset.
+		// Returns the pass-the-end position if not found.
+		// FIXME: improve efficiency
+		uint32_t find_first_unset(rblock const& b, uint32_t offset) {
+			uint32_t entry = offset / BITS_PER_ARRAY_ENTRY;
+			uint32_t nr_entries = b.nr_entries();
+
+			if (entry >= nr_entries)
+				return entry * BITS_PER_ARRAY_ENTRY;
+
+			uint32_t idx = offset % BITS_PER_ARRAY_ENTRY;
+			uint64_t v = b.get(entry++);
+			while (all_bits_set(v, idx) && entry < nr_entries) {
+				v = b.get(entry++);
+				idx = 0;
+			}
+
+			if (all_bits_set(v, idx)) // not found
+				return entry * BITS_PER_ARRAY_ENTRY;
+
+			return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + count_leading_bits(v, idx);
+		}
+
+		// Returns true if all the bits beyond the position are set.
+		bool all_bits_set(uint64_t v, uint32_t offset) {
+			return (v >> offset) == (numeric_limits<uint64_t>::max() >> offset);
+		}
+
+		// Counts the number of leading 1's in the given value, starts from the offset
+		// FIXME: improve efficiency
+		uint32_t count_leading_bits(uint64_t v, uint32_t offset) {
+			uint32_t count = 0;
+
+			v >>= offset;
+			while (v & 0x1) {
+				v >>= 1;
+				count++;
+			}
+
+			return count;
+		}
+
+		block_manager::ptr bm_;
+		uint64_traits::ref_counter rc_;
+
+		const uint32_t BITS_PER_ARRAY_ENTRY;
+	};
+}
+
+//----------------------------------------------------------------
+
+block_dumper::ptr
+dbg::create_bitset_block_dumper() {
+	return block_dumper::ptr(new bitset_block_dumper());
+}
+
+//----------------------------------------------------------------

dbg-lib/bitset_block_dumper.h

@@ -0,0 +1,14 @@
+#ifndef DBG_BITSET_BLOCK_DUMPER
+#define DBG_BITSET_BLOCK_DUMPER
+
+#include "dbg-lib/block_dumper.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	block_dumper::ptr create_bitset_block_dumper();
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/block_dumper.h

@@ -0,0 +1,21 @@
+#ifndef DBG_BLOCK_DUMPER_H
+#define DBG_BLOCK_DUMPER_H
+
+#include "persistent-data/block.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	class block_dumper {
+	public:
+		typedef std::shared_ptr<block_dumper> ptr;
+
+		// pass the read_ref by reference since the caller already held the ref-count
+		virtual void show(persistent_data::block_manager::read_ref &rr,
+				  std::ostream &out) = 0;
+	};
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/btree_node_dumper.h

@@ -0,0 +1,57 @@
+#ifndef DBG_BTREE_NODE_DUMPER
+#define DBG_BTREE_NODE_DUMPER
+
+#include "dbg-lib/block_dumper.h"
+#include "dbg-lib/simple_show_traits.h"
+#include "persistent-data/data-structures/btree.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	using persistent_data::block_manager;
+	using persistent_data::btree_detail::node_ref;
+
+	template <typename ShowTraits>
+	class btree_node_dumper : public block_dumper {
+	public:
+		virtual void show(block_manager::read_ref &rr, std::ostream &out) {
+			node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
+			if (n.get_type() == INTERNAL)
+				btree_node_dumper<uint64_show_traits>::show_node(n, out);
+			else {
+				node_ref<typename ShowTraits::value_trait> n = btree_detail::to_node<typename ShowTraits::value_trait>(rr);
+				show_node(n, out);
+			}
+		}
+
+		static void show_node(node_ref<typename ShowTraits::value_trait> n, std::ostream &out) {
+			formatter::ptr f = create_xml_formatter();
+
+			field(*f, "csum", n.get_checksum());
+			field(*f, "blocknr", n.get_block_nr());
+			field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf");
+			field(*f, "nr_entries", n.get_nr_entries());
+			field(*f, "max_entries", n.get_max_entries());
+			field(*f, "value_size", n.get_value_size());
+
+			for (unsigned i = 0; i < n.get_nr_entries(); i++) {
+				formatter::ptr f2 = create_xml_formatter();
+				field(*f2, "key", n.key_at(i));
+				ShowTraits::show(f2, "value", n.value_at(i));
+				f->child(boost::lexical_cast<std::string>(i), f2);
+			}
+
+			f->output(out, 0);
+		}
+	};
+
+	template <typename ShowTraits>
+	block_dumper::ptr
+	create_btree_node_dumper() {
+		return block_dumper::ptr(new btree_node_dumper<ShowTraits>());
+	}
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/command_interpreter.cc

@@ -0,0 +1,88 @@
+#include "dbg-lib/command_interpreter.h"
+
+using namespace dbg;
+using namespace std;
+
+//----------------------------------------------------------------
+
+namespace {
+	class command_interpreter_impl : public command_interpreter {
+	public:
+		typedef std::shared_ptr<command_interpreter> ptr;
+
+		command_interpreter_impl(std::istream &in, std::ostream &out)
+			: in_(in),
+			  out_(out),
+			  exit_(false) {
+		}
+
+		void register_command(std::string const &str, command::ptr cmd) {
+			commands_.insert(make_pair(str, cmd));
+		}
+
+		void enter_main_loop() {
+			while (!exit_)
+				do_once();
+		}
+
+		void exit_main_loop() {
+			exit_ = true;
+		}
+
+	private:
+		void do_once();
+
+		std::istream &in_;
+		std::ostream &out_;
+		std::map <std::string, command::ptr> commands_;
+		bool exit_;
+	};
+
+	//--------------------------------
+
+	strings read_input(std::istream &in)
+	{
+		using namespace boost::algorithm;
+
+		std::string input;
+		getline(in, input);
+
+		strings toks;
+		split(toks, input, is_any_of(" \t"), token_compress_on);
+
+		return toks;
+	}
+}
+
+//----------------------------------------------------------------
+
+void command_interpreter_impl::do_once()
+{
+	if (in_.eof())
+		throw runtime_error("input closed");
+
+	out_ << "> ";
+	strings args = read_input(in_);
+
+	std::map<std::string, command::ptr>::iterator it;
+	it = commands_.find(args[0]);
+	if (it == commands_.end())
+		out_ << "Unrecognised command" << endl;
+	else {
+		try {
+			it->second->exec(args, out_);
+		} catch (std::exception &e) {
+			cerr << e.what() << endl;
+		}
+	}
+}
+
+//----------------------------------------------------------------
+
+command_interpreter::ptr
+dbg::create_command_interpreter(std::istream &in, std::ostream &out)
+{
+	return command_interpreter::ptr(new command_interpreter_impl(in, out));
+}
+
+//----------------------------------------------------------------

dbg-lib/command_interpreter.h

@@ -0,0 +1,40 @@
+#ifndef DBG_COMMAND_INTERPRETER
+#define DBG_COMMAND_INTERPRETER
+
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/split.hpp>
+
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	typedef std::vector<std::string> strings;
+
+	class command {
+	public:
+		typedef std::shared_ptr<command> ptr;
+
+		virtual ~command() {}
+		virtual void exec(strings const &args, std::ostream &out) = 0;
+	};
+
+	class command_interpreter {
+	public:
+		typedef std::shared_ptr<command_interpreter> ptr;
+
+		virtual void register_command(std::string const &str, command::ptr cmd) = 0;
+		virtual void enter_main_loop() = 0;
+		virtual void exit_main_loop() = 0;
+	};
+
+	command_interpreter::ptr
+	create_command_interpreter(std::istream &in, std::ostream &out);
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/commands.cc

@@ -0,0 +1,79 @@
+#include "dbg-lib/commands.h"
+#include "persistent-data/block.h"
+
+#include <boost/lexical_cast.hpp>
+#include <stdexcept>
+
+using namespace dbg;
+using namespace persistent_data;
+using namespace std;
+
+//----------------------------------------------------------------
+
+namespace {
+	class hello : public dbg::command {
+		virtual void exec(dbg::strings const &args, ostream &out) {
+			out << "Hello, world!" << endl;
+		}
+	};
+
+	class exit_handler : public dbg::command {
+	public:
+		exit_handler(command_interpreter::ptr interpreter)
+			: interpreter_(interpreter) {
+		}
+
+		virtual void exec(dbg::strings const &args, ostream &out) {
+			out << "Goodbye!" << endl;
+			interpreter_->exit_main_loop();
+		}
+
+	private:
+		command_interpreter::ptr interpreter_;
+	};
+
+	class block_handler : public dbg::command {
+	public:
+		block_handler(block_manager::ptr bm, block_dumper::ptr dumper)
+			: bm_(bm), dumper_(dumper) {
+		}
+
+		virtual void exec(dbg::strings const &args, ostream &out) {
+			if (args.size() != 2)
+				throw runtime_error("incorrect number of arguments");
+
+			block_address block = boost::lexical_cast<block_address>(args[1]);
+
+			// no checksum validation for debugging purpose
+			block_manager::read_ref rr = bm_->read_lock(block);
+
+			dumper_->show(rr, out);
+		}
+
+	private:
+		block_manager::ptr bm_;
+		block_dumper::ptr dumper_;
+	};
+}
+
+//----------------------------------------------------------------
+
+command::ptr
+dbg::create_hello_handler()
+{
+	return command::ptr(new hello());
+}
+
+command::ptr
+dbg::create_exit_handler(command_interpreter::ptr interp)
+{
+	return command::ptr(new exit_handler(interp));
+}
+
+command::ptr
+dbg::create_block_handler(block_manager::ptr bm, block_dumper::ptr dumper)
+{
+	return command::ptr(new block_handler(bm, dumper));
+}
+
+//----------------------------------------------------------------

dbg-lib/commands.h

@@ -0,0 +1,24 @@
+#ifndef DBG_COMMANDS_H
+#define DBG_COMMANDS_H
+
+#include "dbg-lib/command_interpreter.h"
+#include "dbg-lib/block_dumper.h"
+#include "persistent-data/block.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	dbg::command::ptr
+	create_hello_handler();
+
+	dbg::command::ptr
+	create_exit_handler(dbg::command_interpreter::ptr interp);
+
+	dbg::command::ptr
+	create_block_handler(persistent_data::block_manager::ptr bm,
+			     dbg::block_dumper::ptr dumper);
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/index_block_dumper.cc

@@ -0,0 +1,51 @@
+#include "dbg-lib/index_block_dumper.h"
+#include "dbg-lib/output_formatter.h"
+#include "dbg-lib/sm_show_traits.h"
+#include "persistent-data/space-maps/disk_structures.h"
+
+using namespace dbg;
+
+//----------------------------------------------------------------
+
+namespace {
+	using persistent_data::block_manager;
+
+	class index_block_dumper : public dbg::block_dumper {
+	public:
+		virtual void show(block_manager::read_ref &rr, std::ostream &out) {
+			sm_disk_detail::metadata_index const *mdi =
+				reinterpret_cast<sm_disk_detail::metadata_index const *>(rr.data());
+			show_metadata_index(mdi, sm_disk_detail::MAX_METADATA_BITMAPS, out);
+		}
+
+	private:
+		void show_metadata_index(sm_disk_detail::metadata_index const *mdi, unsigned nr_indexes, std::ostream &out) {
+			formatter::ptr f = create_xml_formatter();
+			field(*f, "csum", to_cpu<uint32_t>(mdi->csum_));
+			field(*f, "padding", to_cpu<uint32_t>(mdi->padding_));
+			field(*f, "blocknr", to_cpu<uint64_t>(mdi->blocknr_));
+
+			sm_disk_detail::index_entry ie;
+			for (unsigned i = 0; i < nr_indexes; i++) {
+				sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie);
+
+				if (!ie.blocknr_ && !ie.nr_free_ && !ie.none_free_before_)
+					continue;
+
+				formatter::ptr f2 = create_xml_formatter();
+				index_entry_show_traits::show(f2, "value", ie);
+				f->child(boost::lexical_cast<string>(i), f2);
+			}
+			f->output(out, 0);
+		}
+	};
+}
+
+//----------------------------------------------------------------
+
+block_dumper::ptr
+dbg::create_index_block_dumper() {
+	return block_dumper::ptr(new index_block_dumper());
+}
+
+//----------------------------------------------------------------

dbg-lib/index_block_dumper.h

@@ -0,0 +1,10 @@
+#include "dbg-lib/block_dumper.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	block_dumper::ptr
+	create_index_block_dumper();
+}
+
+//----------------------------------------------------------------

dbg-lib/output_formatter.cc

@@ -0,0 +1,86 @@
+#include <string>
+
+#include "dbg-lib/output_formatter.h"
+
+using namespace dbg;
+using namespace std;
+
+//----------------------------------------------------------------
+
+namespace {
+	class abstract_formatter : public formatter {
+		typedef boost::optional<std::string> maybe_string;
+
+		void field(std::string const &name, std::string const &value) {
+			fields_.push_back(field_type(name, value));
+		}
+
+		void child(std::string const &name, formatter::ptr t) {
+			children_.push_back(field_type(name, t));
+		}
+
+	protected:
+		typedef boost::variant<std::string, ptr> value;
+		typedef boost::tuple<std::string, value> field_type;
+
+		std::vector<field_type> fields_;
+		std::vector<field_type> children_;
+	};
+
+	class xml_formatter : public abstract_formatter {
+	public:
+		void output(std::ostream &out, int depth = 0,
+			    boost::optional<std::string> name = boost::none);
+	};
+
+	void indent(int depth, std::ostream &out) {
+		for (int i = 0; i < depth * 2; i++)
+			out << ' ';
+	}
+}
+
+//----------------------------------------------------------------
+
+void xml_formatter::output(std::ostream &out,
+			   int depth,
+			   boost::optional<std::string> name) {
+	indent(depth, out);
+	out << "<fields";
+	if (name && (*name).length())
+		out << " id=\"" << *name << "\"";
+
+	/* output non-child fields */
+	std::vector<field_type>::const_iterator it;
+	for (it = fields_.begin(); it != fields_.end(); ++it) {
+		if (string const *s = boost::get<string>(&it->get<1>())) {
+			out << " " << it->get<0>() << "=\"" << *s << "\"";
+		}
+	}
+
+	if (children_.size() == 0) {
+		out << " />" << endl;
+		return;
+	}
+
+	/* output child fields */
+	out << ">" << endl;
+	for (it = children_.begin(); it != children_.end(); ++it) {
+		if (!boost::get<string>(&it->get<1>())) {
+			formatter::ptr f = boost::get<formatter::ptr>(it->get<1>());
+			f->output(out, depth + 1, it->get<0>());
+		}
+	}
+
+	indent(depth, out);
+	out << "</fields>" << endl;
+}
+
+//----------------------------------------------------------------
+
+formatter::ptr
+dbg::create_xml_formatter()
+{
+	return formatter::ptr(new xml_formatter());
+}
+
+//----------------------------------------------------------------

dbg-lib/output_formatter.h

@@ -0,0 +1,38 @@
+#ifndef DBG_OUTPUT_FORMATTER_H
+#define DBG_OUTPUT_FORMATTER_H
+
+#include <boost/lexical_cast.hpp>
+#include <boost/optional.hpp>
+#include <boost/tuple/tuple.hpp>
+#include <boost/variant.hpp>
+
+#include <memory>
+#include <vector>
+#include <string>
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	class formatter {
+	public:
+		typedef std::shared_ptr<formatter> ptr;
+
+		virtual ~formatter() {}
+		virtual void field(std::string const &name, std::string const &value) = 0;
+		virtual void child(std::string const &name, formatter::ptr t) = 0;
+		virtual void output(std::ostream &out, int depth = 0,
+				    boost::optional<std::string> name = boost::none) = 0;
+	};
+
+	template <typename T>
+	void
+	field(formatter &t, std::string const &name, T const &value) {
+		t.field(name, boost::lexical_cast<std::string>(value));
+	}
+
+	formatter::ptr create_xml_formatter();
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/simple_show_traits.cc

@@ -0,0 +1,19 @@
+#include "dbg-lib/simple_show_traits.h"
+
+using namespace dbg;
+
+//----------------------------------------------------------------
+
+void
+uint32_show_traits::show(formatter::ptr f, string const &key, uint32_t const &value)
+{
+	field(*f, key, boost::lexical_cast<string>(value));
+}
+
+void
+uint64_show_traits::show(formatter::ptr f, string const &key, uint64_t const &value)
+{
+	field(*f, key, boost::lexical_cast<string>(value));
+}
+
+//----------------------------------------------------------------

dbg-lib/simple_show_traits.h

@@ -0,0 +1,27 @@
+#ifndef DBG_SIMPLE_SHOW_TRAITS_H
+#define DBG_SIMPLE_SHOW_TRAITS_H
+
+#include "dbg-lib/output_formatter.h"
+#include "persistent-data/data-structures/simple_traits.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	class uint32_show_traits {
+	public:
+		typedef persistent_data::uint32_traits value_trait;
+
+		static void show(dbg::formatter::ptr f, std::string const &key, uint32_t const &value);
+	};
+
+	class uint64_show_traits {
+	public:
+		typedef persistent_data::uint64_traits value_trait;
+
+		static void show(dbg::formatter::ptr f, std::string const &key, uint64_t const &value);
+	};
+}
+
+//----------------------------------------------------------------
+
+#endif

dbg-lib/sm_show_traits.cc

@@ -0,0 +1,27 @@
+#include "dbg-lib/sm_show_traits.h"
+
+using namespace dbg;
+using namespace std;
+
+//----------------------------------------------------------------
+
+void
+index_entry_show_traits::show(formatter::ptr f, string const &key,
+			      persistent_data::sm_disk_detail::index_entry const &value)
+{
+	field(*f, "blocknr", value.blocknr_);
+	field(*f, "nr_free", value.nr_free_);
+	field(*f, "none_free_before", value.none_free_before_);
+}
+
+void
+sm_root_show_traits::show(formatter::ptr f, string const &key,
+			  persistent_data::sm_disk_detail::sm_root const &value)
+{
+	field(*f, "nr_blocks", value.nr_blocks_);
+	field(*f, "nr_allocated", value.nr_allocated_);
+	field(*f, "bitmap_root", value.bitmap_root_);
+	field(*f, "ref_count_root", value.ref_count_root_);
+}
+
+//----------------------------------------------------------------

dbg-lib/sm_show_traits.h

@@ -0,0 +1,29 @@
+#ifndef DBG_SM_SHOW_TRAITS_H
+#define DBG_SM_SHOW_TRAITS_H
+
+#include "dbg-lib/output_formatter.h"
+#include "persistent-data/space-maps/disk_structures.h"
+
+//----------------------------------------------------------------
+
+namespace dbg {
+	class index_entry_show_traits {
+	public:
+		typedef persistent_data::sm_disk_detail::index_entry_traits value_trait;
+
+		static void show(dbg::formatter::ptr f, std::string const &key,
+				 persistent_data::sm_disk_detail::index_entry const &value);
+	};
+
+	class sm_root_show_traits {
+	public:
+		typedef persistent_data::sm_disk_detail::sm_root_traits value_trait;
+
+		static void show(dbg::formatter::ptr f, std::string const &key,
+				 persistent_data::sm_disk_detail::sm_root const &value);
+	};
+}
+
+//----------------------------------------------------------------
+
+#endif

era/commands.h

@@ -34,6 +34,18 @@ namespace era {
 		virtual int run(int argc, char **argv);
 	};
 
+	//------------------------------------------------------
+
+	class era_debug_cmd : public base::command {
+	public:
+		era_debug_cmd();
+
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	//------------------------------------------------------
+
 	void register_era_commands(base::application &app);
 }
 

era/devel_commands.cc

@@ -0,0 +1,14 @@
+#include "era/commands.h"
+
+using namespace base;
+using namespace era;
+
+//----------------------------------------------------------------
+
+void
+era::register_era_commands(base::application &app)
+{
+	app.add_cmd(command::ptr(new era_debug_cmd));
+}
+
+//----------------------------------------------------------------

era/era_debug.cc

@@ -0,0 +1,228 @@
+// Copyright (C) 2012 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include <boost/lexical_cast.hpp>
+#include <getopt.h>
+#include <iostream>
+#include <string>
+
+#include "dbg-lib/array_block_dumper.h"
+#include "dbg-lib/btree_node_dumper.h"
+#include "dbg-lib/bitset_block_dumper.h"
+#include "dbg-lib/command_interpreter.h"
+#include "dbg-lib/commands.h"
+#include "dbg-lib/index_block_dumper.h"
+#include "dbg-lib/output_formatter.h"
+#include "dbg-lib/sm_show_traits.h"
+#include "persistent-data/file_utils.h"
+#include "persistent-data/space-maps/disk_structures.h"
+#include "era/commands.h"
+#include "era/metadata.h"
+#include "version.h"
+
+using namespace dbg;
+using namespace persistent_data;
+using namespace std;
+using namespace era;
+
+//----------------------------------------------------------------
+
+namespace {
+	class help : public dbg::command {
+		virtual void exec(strings const &args, ostream &out) {
+			out << "Commands:" << endl
+			    << "  superblock [block#]" << endl
+			    << "  block_node <block# of array block-tree node>" << endl
+			    << "  bitset_block <block# of bitset block>" << endl
+			    << "  era_block <block# of era array block>" << endl
+			    << "  index_block <block# of metadata space map root>" << endl
+			    << "  writeset_node <block# of writeset tree node>" << endl
+			    << "  exit" << endl;
+		}
+	};
+
+	// for displaying the writeset tree
+	class writeset_show_traits : public era::era_detail_traits {
+	public:
+		typedef era_detail_traits value_trait;
+
+		static void show(formatter::ptr f, string const &key, era_detail const &value) {
+			field(*f, "nr_bits", value.nr_bits);
+			field(*f, "writeset_root", value.writeset_root);
+		}
+	};
+
+	class show_superblock : public dbg::command {
+	public:
+		explicit show_superblock(block_manager::ptr bm)
+			: bm_(bm) {
+		}
+
+		virtual void exec(strings const &args, ostream &out) {
+			if (args.size() > 2)
+				throw runtime_error("incorrect number of arguments");
+
+			block_address b = era::SUPERBLOCK_LOCATION;
+			if (args.size() == 2)
+				b = boost::lexical_cast<block_address>(args[1]);
+			era::superblock sb = read_superblock(bm_, b);
+
+			formatter::ptr f = create_xml_formatter();
+			ostringstream version;
+			field(*f, "csum", sb.csum);
+			field(*f, "flags", sb.flags.encode());
+			field(*f, "blocknr", sb.blocknr);
+			field(*f, "uuid", sb.uuid); // FIXME: delimit, and handle non-printable chars
+			field(*f, "magic", sb.magic);
+			field(*f, "version", sb.version);
+
+			sm_disk_detail::sm_root_disk const *d;
+			sm_disk_detail::sm_root v;
+			{
+				d = reinterpret_cast<sm_disk_detail::sm_root_disk const *>(sb.metadata_space_map_root);
+				sm_disk_detail::sm_root_traits::unpack(*d, v);
+				formatter::ptr f2 = create_xml_formatter();
+				sm_root_show_traits::show(f2, "value", v);
+				f->child("metadata_space_map_root", f2);
+			}
+
+			field(*f, "data_block_size", sb.data_block_size);
+			field(*f, "metadata_block_size", sb.metadata_block_size);
+			field(*f, "nr_blocks", sb.nr_blocks);
+
+			field(*f, "current_era", sb.current_era);
+			{
+				formatter::ptr f2 = create_xml_formatter();
+				writeset_show_traits::show(f2, "value", sb.current_detail);
+				f->child("current_writeset", f2);
+			}
+
+			field(*f, "writeset_tree_root", sb.writeset_tree_root);
+			field(*f, "era_array_root", sb.era_array_root);
+
+			if (sb.metadata_snap)
+				field(*f, "metadata_snap", *sb.metadata_snap);
+
+			f->output(out, 0);
+		}
+
+	private:
+		block_manager::ptr bm_;
+	};
+
+	//--------------------------------
+
+	template <typename ShowTraits>
+	dbg::command::ptr
+	create_btree_node_handler(block_manager::ptr bm) {
+		return create_block_handler(bm, create_btree_node_dumper<ShowTraits>());
+	}
+
+	template <typename ShowTraits>
+	dbg::command::ptr
+	create_array_block_handler(block_manager::ptr bm,
+				   typename ShowTraits::value_trait::ref_counter rc) {
+		return create_block_handler(bm, create_array_block_dumper<ShowTraits>(rc));
+	}
+
+	dbg::command::ptr
+	create_bitset_block_handler(block_manager::ptr bm) {
+		return create_block_handler(bm, create_bitset_block_dumper());
+	}
+
+	dbg::command::ptr
+	create_index_block_handler(block_manager::ptr bm) {
+		return create_block_handler(bm, create_index_block_dumper());
+	}
+
+	int debug(string const &path) {
+		using dbg::command;
+
+		try {
+			block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY);
+			transaction_manager::ptr null_tm = open_tm(bm, era::SUPERBLOCK_LOCATION);
+			command_interpreter::ptr interp = create_command_interpreter(cin, cout);
+			interp->register_command("hello", create_hello_handler());
+			interp->register_command("superblock", command::ptr(new show_superblock(bm)));
+			interp->register_command("block_node", create_btree_node_handler<uint64_show_traits>(bm));
+			interp->register_command("bitset_block", create_bitset_block_handler(bm));
+			interp->register_command("era_block", create_array_block_handler<uint32_show_traits>(bm,
+					uint32_traits::ref_counter()));
+			interp->register_command("index_block", create_index_block_handler(bm));
+			interp->register_command("writeset_node", create_btree_node_handler<writeset_show_traits>(bm));
+			interp->register_command("help", command::ptr(new help));
+			interp->register_command("exit", create_exit_handler(interp));
+			interp->enter_main_loop();
+
+		} catch (std::exception &e) {
+			cerr << e.what();
+			return 1;
+		}
+
+		return 0;
+	}
+}
+
+//----------------------------------------------------------------
+
+era_debug_cmd::era_debug_cmd()
+	: command("era_debug")
+{
+}
+
+void
+era_debug_cmd::usage(std::ostream &out) const
+{
+	out << "Usage: " << get_name() << " {device|file}" << endl
+	    << "Options:" << endl
+	    << "  {-h|--help}" << endl
+	    << "  {-V|--version}" << endl;
+}
+
+int
+era_debug_cmd::run(int argc, char **argv)
+{
+	int c;
+	const char shortopts[] = "hV";
+	const struct option longopts[] = {
+		{ "help", no_argument, NULL, 'h'},
+		{ "version", no_argument, NULL, 'V'},
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
+		switch(c) {
+		case 'h':
+			usage(cout);
+			return 0;
+
+		case 'V':
+			cerr << THIN_PROVISIONING_TOOLS_VERSION << endl;
+			return 0;
+		}
+	}
+
+	if (argc == optind) {
+		usage(cerr);
+		exit(1);
+	}
+
+	return debug(argv[optind]);
+}
+
+//----------------------------------------------------------------

era/era_detail.h

@@ -22,6 +22,14 @@ namespace era {
 		uint64_t writeset_root;
 	};
 
+	inline bool operator==(era_detail const& lhs, era_detail const& rhs) {
+		return lhs.nr_bits == rhs.nr_bits && lhs.writeset_root == rhs.writeset_root;
+	}
+
+	inline bool operator!=(era_detail const& lhs, era_detail const& rhs) {
+		return !(lhs == rhs);
+	}
+
 	struct era_detail_ref_counter {
 		era_detail_ref_counter(persistent_data::transaction_manager::ptr tm)
 			: tm_(tm) {
@@ -31,7 +39,7 @@ namespace era {
 			tm_->get_sm()->inc(d.writeset_root);
 		}
 
-		void dec(persistent_data::block_address b) {
+		void dec(era_detail const &d) {
 			// I don't think we ever do this in the tools
 			throw std::runtime_error("not implemented");
 		}

era/era_restore.cc

@@ -37,7 +37,7 @@ namespace {
 		bool metadata_touched = false;
 		try {
 			block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE);
-			file_utils::check_file_exists(*fs.output);
+			file_utils::check_file_exists(*fs.input);
 			metadata_touched = true;
 			metadata::ptr md(new metadata(bm, metadata::CREATE));
 			emitter::ptr restorer = create_restore_emitter(*md);

era/metadata.cc

@@ -44,7 +44,7 @@ metadata::metadata(block_manager::ptr bm, open_type ot)
 
 metadata::metadata(block_manager::ptr bm, block_address metadata_snap)
 {
-	open_metadata(bm);
+	open_metadata(bm, metadata_snap);
 }
 
 void

era/restore_emitter.cc

@@ -14,6 +14,7 @@ namespace {
 			: md_(md),
 			  in_superblock_(false),
 			  in_writeset_(false),
+			  era_(0),
 			  in_era_array_(false) {
 		}
 

functional-tests/cache-functional-tests.scm

@@ -49,335 +49,13 @@
   ;; to run.
   (define (register-cache-tests) #t)
 
-  ;;;-----------------------------------------------------------
-  ;;; cache_check scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (cache-check v)
-    "cache_check -V"
-    (run-ok-rcv (stdout _) (cache-check "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-check version)
-    "cache_check --version"
-    (run-ok-rcv (stdout _) (cache-check "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-check h)
-    "cache_check -h"
-    (run-ok-rcv (stdout _) (cache-check "-h")
-      (assert-equal cache-check-help stdout)))
-
-  (define-scenario (cache-check help)
-    "cache_check --help"
-    (run-ok-rcv (stdout _) (cache-check "--help")
-      (assert-equal cache-check-help stdout)))
-
-  (define-scenario (cache-check must-specify-metadata)
-    "Metadata file must be specified"
-    (run-fail-rcv (_ stderr) (cache-check)
-      (assert-equal
-        (string-append "No input file provided.\n"
-                       cache-check-help)
-        stderr)))
-
-  (define-scenario (cache-check no-such-metadata)
-    "Metadata file doesn't exist."
-    (let ((bad-path "/arbitrary/filename"))
-     (run-fail-rcv (_ stderr) (cache-check bad-path)
-       (assert-starts-with
-         (string-append bad-path ": No such file or directory")
-         stderr))))
-
-  (define-scenario (cache-check metadata-file-cannot-be-a-directory)
-    "Metadata file must not be a directory"
-    (let ((bad-path "/tmp"))
-     (run-fail-rcv (_ stderr) (cache-check bad-path)
-       (assert-starts-with
-         (string-append bad-path ": Not a block device or regular file")
-         stderr))))
-
-  (define-scenario (cache-check unreadable-metadata)
-    "Metadata file exists, but is unreadable."
-    (with-valid-metadata (md)
-      (run-ok "chmod" "-r" md)
-      (run-fail-rcv (_ stderr) (cache-check md)
-        (assert-starts-with "syscall 'open' failed: Permission denied" stderr))))
-
-  (define-scenario (cache-check fails-with-corrupt-metadata)
-    "Fail with corrupt superblock"
-    (with-corrupt-metadata (md)
-      (run-fail (cache-check md))))
-
-  (define-scenario (cache-check failing-q)
-    "Fail quietly with -q"
-    (with-corrupt-metadata (md)
-      (run-fail-rcv (stdout stderr) (cache-check "-q" md)
-        (assert-eof stdout)
-        (assert-eof stderr))))
-
-  (define-scenario (cache-check failing-quiet)
-    "Fail quietly with --quiet"
-    (with-corrupt-metadata (md)
-      (run-fail-rcv (stdout stderr) (cache-check "--quiet" md)
-        (assert-eof stdout)
-        (assert-eof stderr))))
-
-  (define-scenario (cache-check valid-metadata-passes)
-    "A valid metadata area passes"
-    (with-valid-metadata (md)
-      (run-ok (cache-check md))))
-
-  (define-scenario (cache-check bad-metadata-version)
-    "Invalid metadata version fails"
-    (with-cache-xml (xml)
-      (with-empty-metadata (md)
-        (cache-restore "-i" xml "-o" md "--debug-override-metadata-version" "12345")
-        (run-fail (cache-check md)))))
-
-  (define-scenario (cache-check tiny-metadata)
-    "Prints helpful message in case tiny metadata given"
-    (with-temp-file-sized ((md "cache.bin" 1024))
-      (run-fail-rcv (_ stderr) (cache-check md)
-        (assert-starts-with "Metadata device/file too small.  Is this binary metadata?" stderr))))
-
-  (define-scenario (cache-check spot-accidental-xml-data)
-    "Prints helpful message if XML metadata given"
-    (with-cache-xml (xml)
-      (system (fmt #f "man bash >> " xml))
-      (run-fail-rcv (_ stderr) (cache-check xml)
-        (assert-matches ".*This looks like XML.  cache_check only checks the binary metadata format." stderr))))
-
-  ;;;-----------------------------------------------------------
-  ;;; cache_restore scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (cache-restore v)
-    "print version (-V flag)"
-    (run-ok-rcv (stdout _) (cache-restore "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-restore version)
-    "print version (--version flags)"
-    (run-ok-rcv (stdout _) (cache-restore "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-restore h)
-    "cache_restore -h"
-    (run-ok-rcv (stdout _) (cache-restore "-h")
-      (assert-equal cache-restore-help stdout)))
-
-  (define-scenario (cache-restore help)
-    "cache_restore --help"
-    (run-ok-rcv (stdout _) (cache-restore "--help")
-      (assert-equal cache-restore-help stdout)))
-
-  (define-scenario (cache-restore no-input-file)
-    "forget to specify an input file"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (cache-restore "-o" md)
-        (assert-starts-with "No input file provided." stderr))))
-
-  (define-scenario (cache-restore missing-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (cache-restore "-i no-such-file -o" md)
-        (assert-superblock-all-zeroes md)
-        (assert-starts-with "Couldn't stat file" stderr))))
-
-  (define-scenario (cache-restore garbage-input-file)
-    "the input file is just zeroes"
-    (with-empty-metadata (md)
-      (with-temp-file-sized ((xml "cache.xml" 4096))
-        (run-fail-rcv (_ stderr) (cache-restore "-i" xml "-o" md)
-          (assert-superblock-all-zeroes md)))))
-
-  (define-scenario (cache-restore missing-output-file)
-    "the output file can't be found"
-    (with-cache-xml (xml)
-      (run-fail-rcv (_ stderr) (cache-restore "-i" xml)
-        (assert-starts-with "No output file provided." stderr))))
-
-  (define-scenario (cache-restore tiny-output-file)
-    "Fails if the output file is too small."
-    (with-temp-file-sized ((md "cache.bin" (* 1024 4)))
-      (with-cache-xml (xml)
-        (run-fail-rcv (_ stderr) (cache-restore "-i" xml "-o" md)
-          (assert-starts-with cache-restore-outfile-too-small-text stderr)))))
-
-  (define-scenario (cache-restore successfully-restores)
-    "Restore succeeds."
-    (with-empty-metadata (md)
-      (with-cache-xml (xml)
-        (run-ok (cache-restore "-i" xml "-o" md)))))
-
-  (define-scenario (cache-restore q)
-    "cache_restore accepts -q"
-    (with-empty-metadata (md)
-      (with-cache-xml (xml)
-        (run-ok-rcv (stdout stderr) (cache-restore "-i" xml "-o" md "-q")
-          (assert-eof stdout)
-          (assert-eof stderr)))))
-
-  (define-scenario (cache-restore quiet)
-    "cache_restore accepts --quiet"
-    (with-empty-metadata (md)
-      (with-cache-xml (xml)
-        (run-ok-rcv (stdout stderr) (cache-restore "-i" xml "-o" md "--quiet")
-          (assert-eof stdout)
-          (assert-eof stderr)))))
-
-  (define-scenario (cache-restore override-metadata-version)
-    "we can set any metadata version"
-    (with-empty-metadata (md)
-      (with-cache-xml (xml)
-        (run-ok
-          (cache-restore "-i" xml "-o" md "--debug-override-metadata-version 10298")))))
-
-  (define-scenario (cache-restore omit-clean-shutdown)
-    "accepts --omit-clean-shutdown"
-    (with-empty-metadata (md)
-      (with-cache-xml (xml)
-        (run-ok
-          (cache-restore "-i" xml "-o" md "--omit-clean-shutdown")))))
-
   ;;;-----------------------------------------------------------
   ;;; cache_dump scenarios
   ;;;-----------------------------------------------------------
 
-  (define-scenario (cache-dump v)
-    "print version (-V flag)"
-    (run-ok-rcv (stdout _) (cache-dump "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-dump version)
-    "print version (--version flags)"
-    (run-ok-rcv (stdout _) (cache-dump "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-dump h)
-    "cache_dump -h"
-    (run-ok-rcv (stdout _) (cache-dump "-h")
-      (assert-equal cache-dump-help stdout)))
-
-  (define-scenario (cache-dump help)
-    "cache_dump --help"
-    (run-ok-rcv (stdout _) (cache-dump "--help")
-      (assert-equal cache-dump-help stdout)))
-
-  (define-scenario (cache-dump missing-input-file)
-    "Fails with missing input file."
-    (run-fail-rcv (stdout stderr) (cache-dump)
-      (assert-starts-with "No input file provided." stderr)))
-
   (define-scenario (cache-dump small-input-file)
     "Fails with small input file"
     (with-temp-file-sized ((md "cache.bin" 512))
       (run-fail
         (cache-dump md))))
-
-  (define-scenario (cache-dump restore-is-noop)
-    "cache_dump followed by cache_restore is a noop."
-    (with-valid-metadata (md)
-      (run-ok-rcv (d1-stdout _) (cache-dump md)
-        (with-temp-file-containing ((xml "cache.xml" d1-stdout))
-          (run-ok (cache-restore "-i" xml "-o" md))
-          (run-ok-rcv (d2-stdout _) (cache-dump md)
-            (assert-equal d1-stdout d2-stdout))))))
-
-  ;;;-----------------------------------------------------------
-  ;;; cache_metadata_size scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (cache-metadata-size v)
-    "cache_metadata_size -V"
-    (run-ok-rcv (stdout _) (cache-metadata-size "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-metadata-size version)
-    "cache_metadata_size --version"
-    (run-ok-rcv (stdout _) (cache-metadata-size "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (cache-metadata-size h)
-    "cache_metadata_size -h"
-    (run-ok-rcv (stdout _) (cache-metadata-size "-h")
-      (assert-equal cache-metadata-size-help stdout)))
-
-  (define-scenario (cache-metadata-size help)
-    "cache_metadata_size --help"
-    (run-ok-rcv (stdout _) (cache-metadata-size "--help")
-      (assert-equal cache-metadata-size-help stdout)))
-
-  (define-scenario (cache-metadata-size no-args)
-    "No arguments specified causes fail"
-    (run-fail-rcv (_ stderr) (cache-metadata-size)
-      (assert-equal "Please specify either --device-size and --block-size, or --nr-blocks."
-                    stderr)))
-
-  (define-scenario (cache-metadata-size device-size-only)
-    "Just --device-size causes fail"
-    (run-fail-rcv (_ stderr) (cache-metadata-size "--device-size" (to-bytes (meg 100)))
-      (assert-equal "If you specify --device-size you must also give --block-size."
-                    stderr)))
-
-  (define-scenario (cache-metadata-size block-size-only)
-    "Just --block-size causes fail"
-    (run-fail-rcv (_ stderr) (cache-metadata-size "--block-size" 128)
-      (assert-equal "If you specify --block-size you must also give --device-size."
-                    stderr)))
-
-  (define-scenario (cache-metadata-size conradictory-info-fails)
-    "Contradictory info causes fail"
-    (run-fail-rcv (_ stderr) (cache-metadata-size "--device-size 102400 --block-size 1000 --nr-blocks 6")
-      (assert-equal "Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size."
-                    stderr)))
-
-  (define-scenario (cache-metadata-size all-args-agree)
-    "All args agreeing succeeds"
-    (run-ok-rcv (stdout stderr) (cache-metadata-size "--device-size" 102400 "--block-size" 100 "--nr-blocks" 1024)
-      (assert-equal "8248 sectors" stdout)
-      (assert-eof stderr)))
-
-  (define-scenario (cache-metadata-size nr-blocks-alone)
-    "Just --nr-blocks succeeds"
-    (run-ok-rcv (stdout stderr) (cache-metadata-size "--nr-blocks" 1024)
-      (assert-equal "8248 sectors" stdout)
-      (assert-eof stderr)))
-
-  (define-scenario (cache-metadata-size dev-size-and-block-size-succeeds)
-    "Specifying --device-size with --block-size succeeds"
-    (run-ok-rcv (stdout stderr) (cache-metadata-size "--device-size" 102400 "--block-size" 100)
-      (assert-equal "8248 sectors" stdout)
-      (assert-eof stderr)))
-
-  (define-scenario (cache-metadata-size big-config)
-    "A big configuration succeeds"
-    (run-ok-rcv (stdout stderr) (cache-metadata-size "--nr-blocks 67108864")
-      (assert-equal "3678208 sectors" stdout)
-      (assert-eof stderr)))
-
-  ;;;-----------------------------------------------------------
-  ;;; cache_repair scenarios
-  ;;;-----------------------------------------------------------
-  (define-scenario (cache-repair missing-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (cache-repair "-i no-such-file -o" md)
-        (assert-superblock-all-zeroes md)
-        (assert-starts-with "Couldn't stat path" stderr))))
-
-  (define-scenario (cache-repair garbage-input-file)
-    "the input file is just zeroes"
-    (with-empty-metadata (md1)
-      (with-corrupt-metadata (md2)
-        (run-fail-rcv (_ stderr) (cache-repair "-i" md1 "-o" md2)
-          (assert-superblock-all-zeroes md2)))))
-
-  (define-scenario (cache-repair missing-output-file)
-    "the output file can't be found"
-    (with-cache-xml (xml)
-      (run-fail-rcv (_ stderr) (cache-repair "-i" xml)
-        (assert-starts-with "No output file provided." stderr))))
-
 )

functional-tests/era-functional-tests.scm

@@ -43,153 +43,9 @@
 
   (define (register-era-tests) #t)
 
-  ;;;-----------------------------------------------------------
-  ;;; era_check scenarios
-  ;;;-----------------------------------------------------------
-  (define-scenario (era-check v)
-    "era_check -V"
-    (run-ok-rcv (stdout _) (era-check "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (era-check version)
-    "era_check --version"
-    (run-ok-rcv (stdout _) (era-check "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (era-check h)
-    "era_check -h"
-    (run-ok-rcv (stdout _) (era-check "-h")
-      (assert-equal era-check-help stdout)))
-
-  (define-scenario (era-check help)
-    "era_check --help"
-    (run-ok-rcv (stdout _) (era-check "--help")
-      (assert-equal era-check-help stdout)))
-
-  (define-scenario (era-check no-device-specified)
-    "Fail if no device specified"
-    (run-fail-rcv (_ stderr) (era-check)
-      (assert-starts-with "No input file provided." stderr)))
-
-  (define-scenario (era-check dev-not-exist)
-    "Fail if specified device doesn't exist"
-    (run-fail-rcv (_ stderr) (era-check "/dev/unlikely")
-      (assert-starts-with "/dev/unlikely: No such file or directory" stderr)))
-
-  (define-scenario (era-check dev-is-a-directory)
-    "Fail if given a directory instead of a file or device"
-    (run-fail-rcv (_ stderr) (era-check "/tmp")
-      (assert-starts-with "/tmp: Not a block device or regular file" stderr)))
-
-  (define-scenario (era-check bad-permissions)
-    "Fail if given a device with inadequate access permissions"
-    (with-temp-file-sized ((md "era.bin" (meg 4)))
-      (run-ok "chmod -r" md)
-      (run-fail-rcv (_ stderr) (era-check md)
-        (assert-starts-with "syscall 'open' failed: Permission denied" stderr))))
-
-  (define-scenario (era-check empty-dev)
-    "Fail if given a file of zeroes"
-    (with-empty-metadata (md)
-      (run-fail (era-check md))))
-
-  (define-scenario (era-check quiet)
-    "Fail should give no output if --quiet"
-    (with-empty-metadata (md)
-      (run-fail-rcv (stdout stderr) (era-check "--quiet" md)
-        (assert-eof stdout)
-        (assert-eof stderr))))
-
-  (define-scenario (era-check q)
-    "Fail should give no output if -q"
-    (with-empty-metadata (md)
-      (run-fail-rcv (stdout stderr) (era-check "-q" md)
-        (assert-eof stdout)
-        (assert-eof stderr))))
-
-  (define-scenario (era-check tiny-metadata)
-    "Prints helpful message in case tiny metadata given"
-    (with-temp-file-sized ((md "era.bin" 1024))
-      (run-fail-rcv (_ stderr) (era-check md)
-        (assert-starts-with "Metadata device/file too small.  Is this binary metadata?" stderr))))
-
-  (define-scenario (era-check spot-accidental-xml-data)
-    "Prints helpful message if XML metadata given"
-    (with-era-xml (xml)
-      (system (fmt #f "man bash >> " xml))
-      (run-fail-rcv (_ stderr) (era-check xml)
-        (assert-matches ".*This looks like XML.  era_check only checks the binary metadata format." stderr))))
-
   ;;;-----------------------------------------------------------
   ;;; era_restore scenarios
   ;;;-----------------------------------------------------------
-  (define-scenario (era-restore v)
-    "era_restore -V"
-    (run-ok-rcv (stdout _) (era-restore "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (era-restore version)
-    "era_restore --version"
-    (run-ok-rcv (stdout _) (era-restore "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (era-restore h)
-    "era_restore -h"
-    (run-ok-rcv (stdout _) (era-restore "-h")
-      (assert-equal era-restore-help stdout)))
-
-  (define-scenario (era-restore help)
-    "era_restore --help"
-    (run-ok-rcv (stdout _) (era-restore "--help")
-      (assert-equal era-restore-help stdout)))
-
-  (define-scenario (era-restore input-unspecified)
-    "Fails if no xml specified"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (era-restore "-o" md)
-        (assert-starts-with "No input file provided." stderr))))
-
-  (define-scenario (era-restore missing-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (era-restore "-i no-such-file -o" md)
-        (assert-superblock-all-zeroes md)
-        (assert-starts-with "Couldn't stat file" stderr))))
-
-  (define-scenario (era-restore garbage-input-file)
-    "the input file is just zeroes"
-    (with-empty-metadata (md)
-      (with-temp-file-sized ((xml "era.xml" 4096))
-        (run-fail-rcv (_ stderr) (era-restore "-i " xml "-o" md)
-          (assert-superblock-all-zeroes md)))))
-
-  (define-scenario (era-restore output-unspecified)
-    "Fails if no metadata dev specified"
-    (with-era-xml (xml)
-      (run-fail-rcv (_ stderr) (era-restore "-i" xml)
-        (assert-starts-with "No output file provided." stderr))))
-
-  (define-scenario (era-restore success)
-    "Succeeds with xml and metadata"
-    (with-era-xml (xml)
-      (with-empty-metadata (md)
-        (run-ok (era-restore "-i" xml "-o" md)))))
-
-  (define-scenario (era-restore quiet)
-    "No output with --quiet (succeeding)"
-    (with-era-xml (xml)
-      (with-empty-metadata (md)
-        (run-ok-rcv (stdout stderr) (era-restore "--quiet" "-i" xml "-o" md)
-          (assert-eof stdout)
-          (assert-eof stderr)))))
-
-  (define-scenario (era-restore q)
-    "No output with -q (succeeding)"
-    (with-era-xml (xml)
-      (with-empty-metadata (md)
-        (run-ok-rcv (stdout stderr) (era-restore "-q" "-i" xml "-o" md)
-          (assert-eof stdout)
-          (assert-eof stderr)))))
 
   (define-scenario (era-restore quiet-fail)
     "No output with --quiet (failing)"
@@ -197,7 +53,9 @@
       (with-empty-metadata (md)
         (run-fail-rcv (stdout stderr) (era-restore "--quiet" "-i" bad-xml "-o" md)
           (assert-eof stdout)
-          (assert-starts-with "Couldn't stat file" stderr)))))
+          (assert-starts-with
+            (string-append bad-xml ": No such file or directory")
+            stderr)))))
 
   (define-scenario (era-restore q-fail)
     "No output with --q(failing)"
@@ -205,22 +63,7 @@
       (with-empty-metadata (md)
         (run-fail-rcv (stdout stderr) (era-restore "-q" "-i" bad-xml "-o" md)
           (assert-eof stdout)
-          (assert-starts-with "Couldn't stat file" stderr)))))
-
-  ;;;-----------------------------------------------------------
-  ;;; era_dump scenarios
-  ;;;-----------------------------------------------------------
-  (define-scenario (era-dump small-input-file)
-    "Fails with small input file"
-    (with-temp-file-sized ((md "era.bin" 512))
-      (run-fail (era-dump md))))
-
-  (define-scenario (era-dump restore-is-noop)
-    "era_dump followed by era_restore is a noop."
-    (with-valid-metadata (md)
-      (run-ok-rcv (d1-stdout _) (era-dump md)
-        (with-temp-file-containing ((xml "era.xml" d1-stdout))
-          (run-ok (era-restore "-i" xml "-o" md))
-          (run-ok-rcv (d2-stdout _) (era-dump md)
-            (assert-equal d1-stdout d2-stdout))))))
+          (assert-starts-with
+            (string-append bad-xml ": No such file or directory")
+            stderr)))))
   )

functional-tests/run-tests.scm

@@ -1,10 +1,8 @@
 (import (rnrs)
         (test-runner)
         (cache-functional-tests)
-        (era-functional-tests)
-        (thin-functional-tests))
+        (era-functional-tests))
 
-(register-thin-tests)
 (register-cache-tests)
 (register-era-tests)
 

functional-tests/test-runner.scm

@@ -14,8 +14,7 @@
           (only (srfi s1 lists) break)
           (regex)
           (srfi s8 receive)
-          (temp-file)
-          (thin-functional-tests))
+          (temp-file))
 
   ;;------------------------------------------------
 

functional-tests/thin-functional-tests.scm

@@ -1,635 +0,0 @@
-(library
-  (thin-functional-tests)
-
-  (export register-thin-tests)
-
-  (import
-    (chezscheme)
-    (bcache block-manager)
-    (disk-units)
-    (fmt fmt)
-    (functional-tests)
-    (process)
-    (scenario-string-constants)
-    (temp-file)
-    (thin xml)
-    (srfi s8 receive))
-
-  (define-tool thin-check)
-  (define-tool thin-delta)
-  (define-tool thin-dump)
-  (define-tool thin-restore)
-  (define-tool thin-rmap)
-  (define-tool thin-repair)
-  (define-tool thin-metadata-pack)
-  (define-tool thin-metadata-unpack)
-
-  (define-syntax with-thin-xml
-    (syntax-rules ()
-      ((_ (v) b1 b2 ...)
-       (with-temp-file-containing ((v "thin.xml" (fmt #f (generate-xml 10 1000))))
-                                  b1 b2 ...))))
-
-  (define-syntax with-valid-metadata
-    (syntax-rules ()
-      ((_ (md) b1 b2 ...)
-       (with-temp-file-sized ((md "thin.bin" (meg 4)))
-         (with-thin-xml (xml)
-           (run-ok (thin-restore "-i" xml "-o" md))
-           b1 b2 ...)))))
-
-  ;;; It would be nice if the metadata was at least similar to valid data.
-  ;;; Here I'm just using the start of the ls binary as 'random' data.
-  (define-syntax with-corrupt-metadata
-    (syntax-rules ()
-      ((_ (md) b1 b2 ...)
-       (with-temp-file-sized ((md "thin.bin" (meg 4)))
-         (system (fmt #f "dd if=/usr/bin/ls of=" md " bs=4096 > /dev/null 2>&1"))
-         b1 b2 ...))))
-
-  (define-syntax with-empty-metadata
-    (syntax-rules ()
-      ((_ (md) b1 b2 ...)
-       (with-temp-file-sized ((md "thin.bin" (meg 4)))
-                             b1 b2 ...))))
-
-  (define (damage-superblock md)
-    (system (string-append "dd if=/dev/zero of=" md " bs=4K count=1 conv=notrunc > /dev/null 2>&1")))
-
-  (define-syntax with-damaged-superblock
-    (syntax-rules ()
-      ((_ (md) b1 b2 ...)
-       (with-valid-metadata (md)
-         (damage-superblock md)
-         b1 b2 ...))))
-
-  ;; We have to export something that forces all the initialisation expressions
-  ;; to run.
-  (define (register-thin-tests) #t)
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_check scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (thin-check v)
-    "thin_check -V"
-    (run-ok-rcv (stdout _) (thin-check "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-check version)
-    "thin_check --version"
-    (run-ok-rcv (stdout _) (thin-check "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-check h)
-    "print help (-h)"
-    (run-ok-rcv (stdout _) (thin-check "-h")
-      (assert-equal thin-check-help stdout)))
-
-  (define-scenario (thin-check help)
-    "print help (--help)"
-    (run-ok-rcv (stdout _) (thin-check "--help")
-      (assert-equal thin-check-help stdout)))
-
-  (define-scenario (thin-check bad-option)
-    "Unrecognised option should cause failure"
-    (run-fail (thin-check "--hedgehogs-only")))
-
-  (define-scenario (thin-check incompatible-options auto-repair)
-    "Incompatible options should cause failure"
-    (with-valid-metadata (md)
-      (run-fail (thin-check "--auto-repair" "-m" md))
-      (run-fail (thin-check "--auto-repair" "--override-mapping-root 123" md))
-      (run-fail (thin-check "--auto-repair" "--super-block-only" md))
-      (run-fail (thin-check "--auto-repair" "--skip-mappings" md))
-      (run-fail (thin-check "--auto-repair" "--ignore-non-fatal-errors" md))))
-
-  (define-scenario (thin-check incompatible-options clear-needs-check-flag)
-    "Incompatible options should cause failure"
-    (with-valid-metadata (md)
-      (run-fail (thin-check "--clear-needs-check-flag" "-m" md))
-      (run-fail (thin-check "--clear-needs-check-flag" "--override-mapping-root 123" md))
-      (run-fail (thin-check "--clear-needs-check-flag" "--super-block-only" md))
-      (run-fail (thin-check "--clear-needs-check-flag" "--skip-mappings" md))
-      (run-fail (thin-check "--clear-needs-check-flag" "--ignore-non-fatal-errors" md))))
-
-  (define-scenario (thin-check superblock-only-valid)
-    "--super-block-only check passes on valid metadata"
-    (with-valid-metadata (md)
-      (run-ok (thin-check "--super-block-only" md))))
-
-  (define-scenario (thin-check superblock-only-invalid)
-    "--super-block-only check fails with corrupt metadata"
-    (with-corrupt-metadata (md)
-      (run-fail (thin-check "--super-block-only" md))))
-
-  (define-scenario (thin-check skip-mappings-valid)
-    "--skip-mappings check passes on valid metadata"
-    (with-valid-metadata (md)
-      (run-ok (thin-check "--skip-mappings" md))))
-
-  (define-scenario (thin-check ignore-non-fatal-errors)
-    "--ignore-non-fatal-errors check passes on valid metadata"
-    (with-valid-metadata (md)
-      (run-ok (thin-check "--ignore-non-fatal-errors" md))))
-
-  (define-scenario (thin-check quiet)
-    "--quiet should give no output"
-    (with-valid-metadata (md)
-      (run-ok-rcv (stdout stderr) (thin-check "--quiet" md)
-        (assert-eof stdout)
-        (assert-eof stderr))))
-
-  (define-scenario (thin-check clear-needs-check-flag)
-    "Accepts --clear-needs-check-flag"
-    (with-valid-metadata (md)
-      (run-ok (thin-check "--clear-needs-check-flag" md))))
-
-  (define-scenario (thin-check auto-repair)
-    "Accepts --auto-repair"
-    (with-valid-metadata (md)
-      (run-ok (thin-check "--auto-repair" md))))
-
-  (define-scenario (thin-check tiny-metadata)
-    "Prints helpful message in case tiny metadata given"
-    (with-temp-file-sized ((md "thin.bin" 1024))
-      (run-fail-rcv (_ stderr) (thin-check md)
-        (assert-starts-with "Metadata device/file too small.  Is this binary metadata?" stderr))))
-
-  (define-scenario (thin-check spot-accidental-xml-data)
-    "Prints helpful message if XML metadata given"
-    (with-thin-xml (xml)
-      (system (fmt #f "man bash >> " xml))
-      (run-fail-rcv (_ stderr) (thin-check xml)
-        (assert-matches ".*This looks like XML.  thin_check only checks the binary metadata format." stderr))))
-
-  (define-scenario (thin-check info-fields)
-    "Outputs info fields"
-    (with-valid-metadata (md)
-      (run-ok-rcv (stdout stderr) (thin-check md)
-        (assert-matches ".*TRANSACTION_ID=[0-9]+.*" stdout)
-	(assert-matches ".*METADATA_FREE_BLOCKS=[0-9]+.*" stdout))))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_restore scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (thin-restore print-version-v)
-    "print help (-V)"
-    (run-ok-rcv (stdout _) (thin-restore "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-restore print-version-long)
-    "print help (--version)"
-    (run-ok-rcv (stdout _) (thin-restore "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-restore h)
-    "print help (-h)"
-    (run-ok-rcv (stdout _) (thin-restore "-h")
-      (assert-equal thin-restore-help stdout)))
-
-  (define-scenario (thin-restore help)
-    "print help (-h)"
-    (run-ok-rcv (stdout _) (thin-restore "--help")
-      (assert-equal thin-restore-help stdout)))
-
-  (define-scenario (thin-restore no-input-file)
-    "forget to specify an input file"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-restore "-o" md)
-        (assert-starts-with "No input file provided." stderr))))
-
-  (define-scenario (thin-restore missing-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-restore "-i no-such-file -o" md)
-        (assert-superblock-all-zeroes md)
-        (assert-starts-with "Couldn't stat file" stderr))))
-
-  (define-scenario (thin-restore garbage-input-file)
-    "the input file is just zeroes"
-    (with-empty-metadata (md)
-      (with-temp-file-sized ((xml "thin.xml" 4096))
-        (run-fail-rcv (_ stderr) (thin-restore "-i " xml "-o" md)
-          (assert-superblock-all-zeroes md)))))
-
-  (define-scenario (thin-restore missing-output-file)
-    "the output file can't be found"
-    (with-thin-xml (xml)
-      (run-fail-rcv (_ stderr) (thin-restore "-i " xml)
-        (assert-starts-with "No output file provided." stderr))))
-
-  (define-scenario (thin-restore tiny-output-file)
-    "Fails if the output file is too small."
-    (with-temp-file-sized ((md "thin.bin" 4096))
-      (with-thin-xml (xml)
-        (run-fail-rcv (_ stderr) (thin-restore "-i" xml "-o" md)
-          (assert-starts-with thin-restore-outfile-too-small-text stderr)))))
-
-  (define-scenario (thin-restore q)
-    "thin_restore accepts -q"
-    (with-empty-metadata (md)
-      (with-thin-xml (xml)
-        (run-ok-rcv (stdout _) (thin-restore "-i" xml "-o" md "-q")
-          (assert-eof stdout)))))
-
-  (define-scenario (thin-restore quiet)
-    "thin_restore accepts --quiet"
-    (with-empty-metadata (md)
-      (with-thin-xml (xml)
-        (run-ok-rcv (stdout _) (thin-restore "-i" xml "-o" md "--quiet")
-          (assert-eof stdout)))))
-
-  (define-scenario (thin-restore override transaction-id)
-    "thin_restore obeys the --transaction-id override"
-    (with-empty-metadata (md)
-      (with-thin-xml (xml)
-        (run-ok-rcv (stdout stderr) (thin-restore "--transaction-id 2345" "-i" xml "-o" md)
-          (assert-eof stderr))
-        (run-ok-rcv (stdout stderr) (thin-dump md)
-          (assert-matches ".*transaction=\"2345\"" stdout)))))
-
-  (define-scenario (thin-restore override data-block-size)
-    "thin_restore obeys the --data-block-size override"
-    (with-empty-metadata (md)
-      (with-thin-xml (xml)
-        (run-ok-rcv (stdout stderr) (thin-restore "--data-block-size 8192" "-i" xml "-o" md)
-          (assert-eof stderr))
-        (run-ok-rcv (stdout stderr) (thin-dump md)
-          (assert-matches ".*data_block_size=\"8192\"" stdout)))))
-
-  (define-scenario (thin-restore override nr-data-blocks)
-    "thin_restore obeys the --nr-data-blocks override"
-    (with-empty-metadata (md)
-      (with-thin-xml (xml)
-        (run-ok-rcv (stdout stderr) (thin-restore "--nr-data-blocks 234500" "-i" xml "-o" md)
-          (assert-eof stderr))
-        (run-ok-rcv (stdout stderr) (thin-dump md)
-          (assert-matches ".*nr_data_blocks=\"234500\"" stdout)))))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_dump scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (thin-dump small-input-file)
-    "Fails with small input file"
-    (with-temp-file-sized ((md "thin.bin" 512))
-      (run-fail (thin-dump md))))
-
-  (define-scenario (thin-dump restore-is-noop)
-    "thin_dump followed by thin_restore is a noop."
-    (with-valid-metadata (md)
-      (run-ok-rcv (d1-stdout _) (thin-dump md)
-        (with-temp-file-containing ((xml "thin.xml" d1-stdout))
-          (run-ok (thin-restore "-i" xml "-o" md))
-          (run-ok-rcv (d2-stdout _) (thin-dump md)
-            (assert-equal d1-stdout d2-stdout))))))
-
-  (define-scenario (thin-dump no-stderr)
-    "thin_dump of clean data does not output error messages to stderr"
-    (with-valid-metadata (md)
-      (run-ok-rcv (stdout stderr) (thin-dump md)
-        (assert-eof stderr))))
-
-  (define-scenario (thin-dump override transaction-id)
-    "thin_dump obeys the --transaction-id override"
-    (with-valid-metadata (md)
-      (run-ok-rcv (stdout stderr) (thin-dump "--transaction-id 2345" md)
-        (assert-eof stderr)
-        (assert-matches ".*transaction=\"2345\"" stdout))))
-
-  (define-scenario (thin-dump override data-block-size)
-    "thin_dump obeys the --data-block-size override"
-    (with-valid-metadata (md)
-      (run-ok-rcv (stdout stderr) (thin-dump "--data-block-size 8192" md)
-        (assert-eof stderr)
-        (assert-matches ".*data_block_size=\"8192\"" stdout))))
-
-  (define-scenario (thin-dump override nr-data-blocks)
-    "thin_dump obeys the --nr-data-blocks override"
-    (with-valid-metadata (md)
-      (run-ok-rcv (stdout stderr) (thin-dump "--nr-data-blocks 234500" md)
-        (assert-eof stderr)
-        (assert-matches ".*nr_data_blocks=\"234500\"" stdout))))
-
-  (define-scenario (thin-dump repair-superblock succeeds)
-    "thin_dump can restore a missing superblock"
-    (with-valid-metadata (md)
-      (run-ok-rcv (expected-xml stderr) (thin-dump "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" md)
-        (damage-superblock md)
-        (run-ok-rcv (repaired-xml stderr) (thin-dump "--repair" "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" md)
-          (assert-eof stderr)
-          (assert-equal expected-xml repaired-xml)))))
-
-  (define-scenario (thin-dump repair-superblock missing-transaction-id)
-    "--transaction-id is mandatory if the superblock is damaged"
-    (with-damaged-superblock (md)
-      (run-fail-rcv (_ stderr) (thin-dump "--repair" "--data-block-size=128" "--nr-data-blocks=4096000" md)
-        (assert-matches ".*transaction id.*" stderr))))
-
-  (define-scenario (thin-dump repair-superblock missing-data-block-size)
-    "--data-block-size is mandatory if the superblock is damaged"
-    (with-damaged-superblock (md)
-      (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--nr-data-blocks=4096000" md)
-        (assert-matches ".*data block size.*" stderr))))
-
-  (define-scenario (thin-dump repair-superblock missing-nr-data-blocks)
-    "--nr-data-blocks is mandatory if the superblock is damaged"
-    (with-damaged-superblock (md)
-      (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--data-block-size=128" md)
-        (assert-matches ".*nr data blocks.*" stderr))))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_rmap scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (thin-rmap v)
-    "thin_rmap accepts -V"
-    (run-ok-rcv (stdout _) (thin-rmap "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-rmap version)
-    "thin_rmap accepts --version"
-    (run-ok-rcv (stdout _) (thin-rmap "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-rmap h)
-    "thin_rmap accepts -h"
-    (run-ok-rcv (stdout _) (thin-rmap "-h")
-      (assert-equal thin-rmap-help stdout)))
-
-  (define-scenario (thin-rmap help)
-    "thin_rmap accepts --help"
-    (run-ok-rcv (stdout _) (thin-rmap "--help")
-      (assert-equal thin-rmap-help stdout)))
-
-  (define-scenario (thin-rmap unrecognised-flag)
-    "thin_rmap complains with bad flags."
-    (run-fail (thin-rmap "--unleash-the-hedgehogs")))
-
-  (define-scenario (thin-rmap valid-region-format-should-pass)
-    "thin_rmap with a valid region format should pass."
-    (with-valid-metadata (md)
-      (run-ok
-        (thin-rmap "--region 23..7890" md))))
-
-  (define-scenario (thin-rmap invalid-region-should-fail)
-    "thin_rmap with an invalid region format should fail."
-    (for-each (lambda (pattern)
-                      (with-valid-metadata (md)
-                        (run-fail (thin-rmap "--region" pattern md))))
-              '("23,7890" "23..six" "found..7890" "89..88" "89..89" "89.." "" "89...99")))
-
-  (define-scenario (thin-rmap multiple-regions-should-pass)
-    "thin_rmap should handle multiple regions."
-    (with-valid-metadata (md)
-      (run-ok (thin-rmap "--region 1..23 --region 45..78" md))))
-
-  (define-scenario (thin-rmap handles-junk-input)
-    "Fail gracefully if given nonsense"
-    (with-thin-xml (xml)
-      (run-fail-rcv (_ stderr) (thin-rmap "--region 0..-1" xml)
-          #t)))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_delta scenarios
-  ;;;-----------------------------------------------------------
-  (define-scenario (thin-delta v)
-    "thin_delta accepts -V"
-    (run-ok-rcv (stdout _) (thin-delta "-V")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-delta version)
-    "thin_delta accepts --version"
-    (run-ok-rcv (stdout _) (thin-delta "--version")
-      (assert-equal tools-version stdout)))
-
-  (define-scenario (thin-delta h)
-    "thin_delta accepts -h"
-    (run-ok-rcv (stdout _) (thin-delta "-h")
-      (assert-equal thin-delta-help stdout)))
-
-  (define-scenario (thin-delta help)
-    "thin_delta accepts --help"
-    (run-ok-rcv (stdout _) (thin-delta "--help")
-      (assert-equal thin-delta-help stdout)))
-
-  (define-scenario (thin-delta unrecognised-option)
-    "Unrecognised option should cause failure"
-    (with-valid-metadata (md)
-      (run-fail-rcv (stdout stderr) (thin-delta "--unleash-the-hedgehogs")
-        (assert-matches ".*thin_delta: unrecognized option '--unleash-the-hedgehogs" stderr))))
-
-  (define-scenario (thin-delta snap1-unspecified)
-    "Fails without --snap1 fails"
-    (run-fail-rcv (_ stderr) (thin-delta "--snap2 45 foo")
-      (assert-starts-with "--snap1 not specified." stderr)))
-
-  (define-scenario (thin-delta snap2-unspecified)
-    "Fails without --snap2 fails"
-    (run-fail-rcv (_ stderr) (thin-delta "--snap1 45 foo")
-      (assert-starts-with "--snap2 not specified." stderr)))
-
-  (define-scenario (thin-delta device-unspecified)
-    "Fails if no device given"
-    (run-fail-rcv (_ stderr) (thin-delta "--snap1 45 --snap2 46")
-      (assert-starts-with "No input device provided." stderr)))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_repair scenarios
-  ;;;-----------------------------------------------------------
-  (define-scenario (thin-repair dont-repair-xml)
-    "Fails gracefully if run on XML rather than metadata"
-    (with-thin-xml (xml)
-      (with-empty-metadata (md)
-        (run-fail-rcv (_ stderr) (thin-repair "-i" xml "-o" md)
-          #t))))
-
-  (define-scenario (thin-repair missing-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-repair "-i no-such-file -o" md)
-        (assert-superblock-all-zeroes md)
-        (assert-starts-with "Couldn't stat file" stderr))))
-
-  (define-scenario (thin-repair garbage-input-file)
-    "the input file is just zeroes"
-    (with-empty-metadata (md1)
-      (with-corrupt-metadata (md2)
-        (run-fail-rcv (_ stderr) (thin-repair "-i " md1 "-o" md2)
-          (assert-superblock-all-zeroes md2)))))
-
-  (define-scenario (thin-repair missing-output-file)
-    "the output file can't be found"
-    (with-thin-xml (xml)
-      (run-fail-rcv (_ stderr) (thin-repair "-i " xml)
-        (assert-starts-with "No output file provided." stderr))))
-
-  (define-scenario (thin-repair override transaction-id)
-    "thin_repair obeys the --transaction-id override"
-    (with-valid-metadata (md1)
-      (with-empty-metadata (md2)
-        (run-ok-rcv (stdout stderr) (thin-repair "--transaction-id 2345" "-i" md1 "-o" md2)
-          (assert-eof stderr))
-        (run-ok-rcv (stdout stderr) (thin-dump md2)
-          (assert-matches ".*transaction=\"2345\"" stdout)))))
-
-  (define-scenario (thin-repair override data-block-size)
-    "thin_repair obeys the --data-block-size override"
-    (with-valid-metadata (md1)
-      (with-empty-metadata (md2)
-        (run-ok-rcv (stdout stderr) (thin-repair "--data-block-size 8192" "-i" md1 "-o" md2)
-          (assert-eof stderr))
-        (run-ok-rcv (stdout stderr) (thin-dump md2)
-          (assert-matches ".*data_block_size=\"8192\"" stdout)))))
-
-  (define-scenario (thin-repair override nr-data-blocks)
-    "thin_repair obeys the --nr-data-blocks override"
-    (with-valid-metadata (md1)
-      (with-empty-metadata (md2)
-        (run-ok-rcv (stdout stderr) (thin-repair "--nr-data-blocks 234500" "-i" md1 "-o" md2)
-          (assert-eof stderr))
-        (run-ok-rcv (stdout stderr) (thin-dump md2)
-          (assert-matches ".*nr_data_blocks=\"234500\"" stdout)))))
-
-  (define-scenario (thin-repair superblock succeeds)
-    "thin_repair can restore a missing superblock"
-    (with-valid-metadata (md1)
-      (run-ok-rcv (expected-xml stderr) (thin-dump "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" md1)
-        (damage-superblock md1)
-        (with-empty-metadata (md2)
-          (run-ok-rcv (_ stderr) (thin-repair "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" "-i" md1 "-o" md2)
-            (assert-eof stderr))
-          (run-ok-rcv (repaired-xml stderr) (thin-dump md2)
-            (assert-eof stderr)
-            (assert-equal expected-xml repaired-xml))))))
-
-  (define-scenario (thin-repair superblock missing-transaction-id)
-    "--transaction-id is mandatory if the superblock is damaged"
-    (with-damaged-superblock (md1)
-      (with-empty-metadata (md2)
-        (run-fail-rcv (_ stderr) (thin-repair "--data-block-size=128" "--nr-data-blocks=4096000" "-i" md1 "-o" md2)
-          (assert-matches ".*transaction id.*" stderr)))))
-
-  (define-scenario (thin-repair superblock missing-data-block-size)
-    "--data-block-size is mandatory if the superblock is damaged"
-    (with-damaged-superblock (md1)
-      (with-empty-metadata (md2)
-        (run-fail-rcv (_ stderr) (thin-repair "--transaction-id=5" "--nr-data-blocks=4096000" "-i" md1 "-o" md2)
-          (assert-matches ".*data block size.*" stderr)))))
-
-  (define-scenario (thin-repair superblock missing-nr-data-blocks)
-    "--nr-data-blocks is mandatory if the superblock is damaged"
-    (with-damaged-superblock (md1)
-      (with-empty-metadata (md2)
-        (run-fail-rcv (_ stderr) (thin-repair "--transaction-id=5" "--data-block-size=128" "-i" md1 "-o" md2)
-          (assert-matches ".*nr data blocks.*" stderr)))))
-
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_metadata_pack scenarios
-  ;;;-----------------------------------------------------------
-
-  (define-scenario (thin-metadata-pack version)
-    "accepts --version"
-    (run-ok-rcv (stdout _) (thin-metadata-pack "--version")
-      (assert-equal "thin_metadata_pack 0.9.0-rc2" stdout)))
-
-  (define-scenario (thin-metadata-pack h)
-    "accepts -h"
-    (run-ok-rcv (stdout _) (thin-metadata-pack "-h")
-      (assert-equal thin-metadata-pack-help stdout)))
-
-  (define-scenario (thin-metadata-pack help)
-    "accepts --help"
-    (run-ok-rcv (stdout _) (thin-metadata-pack "--help")
-      (assert-equal thin-metadata-pack-help stdout)))
-
-  (define-scenario (thin-metadata-pack unrecognised-option)
-    "Unrecognised option should cause failure"
-    (with-valid-metadata (md)
-      (run-fail-rcv (stdout stderr) (thin-metadata-pack "--unleash-the-hedgehogs")
-        (assert-starts-with "error: Found argument '--unleash-the-hedgehogs'" stderr)))) 
-
-  (define-scenario (thin-metadata-pack missing-input-file)
-    "the input file wasn't specified"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-metadata-pack "-o " md)
-        (assert-starts-with "error: The following required arguments were not provided:\n    -i <DEV>" stderr))))
-
-  (define-scenario (thin-metadata-pack no-such-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-metadata-pack "-i no-such-file -o" md)
-        (assert-starts-with "Couldn't find input file" stderr))))
-
-  (define-scenario (thin-metadata-pack missing-output-file)
-    "the output file wasn't specified"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-metadata-pack "-i" md)
-        (assert-starts-with "error: The following required arguments were not provided:\n    -o <FILE>" stderr))))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_metadata_unpack scenarios
-  ;;;-----------------------------------------------------------
-  (define-scenario (thin-metadata-unpack version)
-    "accepts --version"
-    (run-ok-rcv (stdout _) (thin-metadata-unpack "--version")
-      (assert-equal "thin_metadata_unpack 0.9.0-rc2" stdout)))
-
-  (define-scenario (thin-metadata-unpack h)
-    "accepts -h"
-    (run-ok-rcv (stdout _) (thin-metadata-unpack "-h")
-      (assert-equal thin-metadata-unpack-help stdout)))
-
-  (define-scenario (thin-metadata-unpack help)
-    "accepts --help"
-    (run-ok-rcv (stdout _) (thin-metadata-unpack "--help")
-      (assert-equal thin-metadata-unpack-help stdout)))
-
-  (define-scenario (thin-metadata-unpack unrecognised-option)
-    "Unrecognised option should cause failure"
-    (with-valid-metadata (md)
-      (run-fail-rcv (stdout stderr) (thin-metadata-unpack "--unleash-the-hedgehogs")
-        (assert-starts-with "error: Found argument '--unleash-the-hedgehogs'" stderr)))) 
-
-  (define-scenario (thin-metadata-unpack missing-input-file)
-    "the input file wasn't specified"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-metadata-unpack "-o " md)
-        (assert-starts-with "error: The following required arguments were not provided:\n    -i <DEV>" stderr))))
-
-  (define-scenario (thin-metadata-unpack no-such-input-file)
-    "the input file can't be found"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-metadata-unpack "-i no-such-file -o" md)
-        (assert-starts-with "Couldn't find input file" stderr))))
-
-  (define-scenario (thin-metadata-unpack missing-output-file)
-    "the output file wasn't specified"
-    (with-empty-metadata (md)
-      (run-fail-rcv (_ stderr) (thin-metadata-unpack "-i" md)
-        (assert-starts-with "error: The following required arguments were not provided:\n    -o <FILE>" stderr))))
-
-  (define-scenario (thin-metadata-unpack garbage-input-file)
-    "the input file is just zeroes"
-    (with-empty-metadata (bad-pack)
-      (run-fail-rcv (_ stderr) (thin-metadata-unpack "-i " bad-pack "-o junk")
-        (assert-starts-with "Not a pack file." stderr))))
-
-  ;;;-----------------------------------------------------------
-  ;;; thin_metadata_pack/unpack end to end scenario
-  ;;;-----------------------------------------------------------)
-  (define-scenario (thin-metadata-pack end-to-end)
-    "pack -> unpack recovers metadata"
-    (let ((pack-file "md.pack"))
-      (with-valid-metadata (md-in)
-        (with-empty-metadata (md-out)
-          (run-ok (thin-metadata-pack "-i" md-in "-o" pack-file))
-          (run-ok (thin-metadata-unpack "-i" pack-file "-o" md-out))
-            (run-ok-rcv (dump1 _) (thin-dump md-in)
-              (run-ok-rcv (dump2 _) (thin-dump md-out)
-                (assert-equal dump1 dump2)))))))
-
-)

persistent-data/block_counter.h

@@ -62,6 +62,10 @@ namespace persistent_data {
 			return stop_on_error_;
 		}
 
+		virtual void clear() {
+			counts_.clear();
+		}
+
 	private:
 		count_map counts_;
 		bool stop_on_error_;
@@ -86,6 +90,11 @@ namespace persistent_data {
 		base::run_set<block_address> const& get_visited() const {
 			return visited_;
 		}
+
+		virtual void clear() {
+			visited_.clear();
+		}
+
 	private:
 		base::run_set<block_address> visited_;
 	};

persistent-data/data-structures/array.h

@@ -104,7 +104,7 @@ namespace persistent_data {
 	template <typename ValueTraits>
 	class array : public array_base {
 	public:
-		class block_ref_counter : public ref_counter<uint64_t> {
+		class block_ref_counter : public persistent_data::ref_counter<uint64_t> {
 		public:
 			block_ref_counter(space_map::ptr sm,
 					  array<ValueTraits> &a)

persistent-data/data-structures/btree-remove.tcc

@@ -145,11 +145,11 @@ namespace persistent_data {
 	{
 		internal_node n = spine.get_node<block_traits>();
 
+		// compact the path if there's only one child
 		if (n.get_nr_entries() == 1) {
 			block_address b = n.value_at(0);
 			read_ref child = tm_.read_lock(b, validator_);
 
-			// FIXME: is it safe?
 			::memcpy(n.raw(), child.data(), read_ref::BLOCK_SIZE);
 
 			tm_.get_sm()->dec(child.get_location());
@@ -341,7 +341,6 @@ namespace persistent_data {
 		if (nr_left < nr_right) {
 			int s = nr_left - target_left;
 
-			// FIXME: signed & unsigned comparison
 			if (s < 0 && nr_center < static_cast<unsigned>(-s)) {
 				// not enough in central node
 				left.move_entries(center, -nr_center);

persistent-data/data-structures/btree.tcc

@@ -338,7 +338,7 @@ namespace persistent_data {
 		unsigned nr_right = rhs.get_nr_entries();
 		unsigned max_entries = get_max_entries();
 
-		if (nr_left - count > max_entries || nr_right - count > max_entries)
+		if (nr_left - count > max_entries || nr_right + count > max_entries)
 			throw runtime_error("too many entries");
 
 		if (count > 0) {
@@ -693,9 +693,15 @@ namespace persistent_data {
 		leaf_node n = spine.template get_node<ValueTraits>();
 		if (need_insert)
 			n.insert_at(index, key[Levels - 1], value);
-		else
-			// FIXME: check if we're overwriting with the same value.
-			n.set_value(index, value);
+		else {
+			typename ValueTraits::value_type old_value = n.value_at(index);
+			if (value != old_value) {
+				// do decrement the old value if it already exists
+				rc_.dec(old_value);
+
+				n.set_value(index, value);
+			}
+		}
 
 		root_ = spine.get_root();
 
@@ -981,11 +987,6 @@ namespace persistent_data {
 		if (i < 0 || leaf.key_at(i) != key)
 			i++;
 
-		// do decrement the old value if it already exists
-		// FIXME: I'm not sure about this, I don't understand the |inc| reference
-		if (static_cast<unsigned>(i) < leaf.get_nr_entries() && leaf.key_at(i) == key && inc) {
-			// dec old entry
-		}
 		*index = i;
 
 		return ((static_cast<unsigned>(i) >= leaf.get_nr_entries()) ||

persistent-data/data-structures/btree_counter.h

@@ -14,12 +14,14 @@ namespace persistent_data {
 		public:
 			typedef btree<Levels, ValueTraits> tree;
 
-			counting_visitor(block_counter &bc, ValueCounter &vc)
+			counting_visitor(block_counter &bc, ValueCounter &vc,
+					 bool ignore_non_fatal = false)
 				: bc_(bc),
 				  vc_(vc),
 				  error_outcome_(bc.stop_on_error() ?
 						 tree::visitor::RETHROW_EXCEPTION :
-						 tree::visitor::EXCEPTION_HANDLED) {
+						 tree::visitor::EXCEPTION_HANDLED),
+				  ignore_non_fatal_(ignore_non_fatal) {
 			}
 
 			virtual bool visit_internal(node_location const &l,
@@ -66,7 +68,7 @@ namespace persistent_data {
 				if (!checker_.check_block_nr(n) ||
 				    !checker_.check_value_size(n) ||
 				    !checker_.check_max_entries(n) ||
-				    !checker_.check_nr_entries(n, l.is_sub_root()) ||
+				    !check_nr_entries(l, n) ||
 				    !checker_.check_ordered_keys(n) ||
 				    !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional<uint64_t>() : l.key))
 					return false;
@@ -83,7 +85,7 @@ namespace persistent_data {
 				if (!checker_.check_block_nr(n) ||
 				    !checker_.check_value_size(n) ||
 				    !checker_.check_max_entries(n) ||
-				    !checker_.check_nr_entries(n, l.is_sub_root()) ||
+				    !check_nr_entries(l, n) ||
 				    !checker_.check_ordered_keys(n) ||
 				    !checker_.check_parent_key(n, l.is_sub_root() ? boost::optional<uint64_t>() : l.key) ||
 				    !checker_.check_leaf_key(n, last_leaf_key_[l.level()]))
@@ -109,11 +111,18 @@ namespace persistent_data {
 				return !seen;
 			}
 
+			template <typename ValueTraits2>
+			bool check_nr_entries(node_location const &loc,
+					      btree_detail::node_ref<ValueTraits2> const &n) {
+				return ignore_non_fatal_ || checker_.check_nr_entries(n, loc.is_sub_root());
+			}
+
 			block_counter &bc_;
 			ValueCounter &vc_;
 			btree_node_checker checker_;
 			boost::optional<uint64_t> last_leaf_key_[Levels];
 			error_outcome error_outcome_;
+			bool ignore_non_fatal_;
 		};
 	}
 
@@ -141,8 +150,9 @@ namespace persistent_data {
 	// walked.  This walk should only be done once you're sure the tree
 	// is not corrupt.
 	template <unsigned Levels, typename ValueTraits, typename ValueCounter>
-	void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc) {
-		btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc);
+	void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc,
+				bool ignore_non_fatal = false) {
+		btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc, ignore_non_fatal);
 		tree.visit_depth_first(v);
 	}
 }

persistent-data/space-maps/disk.cc

@@ -666,9 +666,14 @@ namespace {
 				if (!ie.blocknr_)
 					return;
 
-				block_manager::read_ref rr = tm_.read_lock(ie.blocknr_, bitmap_validator_);
-				if (rr.data())
-					bc_.inc(ie.blocknr_);
+				try {
+					block_manager::read_ref rr = tm_.read_lock(ie.blocknr_, bitmap_validator_);
+					if (rr.data())
+						bc_.inc(ie.blocknr_);
+				} catch (std::exception &e) {
+					if (bc_.stop_on_error())
+						throw;
+				}
 			}
 
 		private:

persistent-data/space-maps/disk_structures.h

@@ -42,6 +42,16 @@ namespace persistent_data {
 			uint32_t none_free_before_;
 		};
 
+		inline bool operator==(index_entry const& lhs, index_entry const& rhs) {
+			// The return value doesn't matter, since the ref-counts of bitmap blocks
+			// are managed by shadow operations.
+			return false;
+		}
+
+		inline bool operator!=(index_entry const& lhs, index_entry const& rhs) {
+			return !(lhs == rhs);
+		}
+
 		struct index_entry_traits {
 			typedef index_entry_disk disk_type;
 			typedef index_entry value_type;

persistent-data/transaction_manager.cc

@@ -37,6 +37,13 @@ transaction_manager::~transaction_manager()
 {
 }
 
+void
+transaction_manager::commit()
+{
+	wipe_shadow_table();
+	bm_->flush();
+}
+
 transaction_manager::write_ref
 transaction_manager::begin(block_address superblock, validator v)
 {

persistent-data/transaction_manager.h

@@ -42,6 +42,8 @@ namespace persistent_data {
 				    space_map::ptr sm);
 		~transaction_manager();
 
+		void commit();
+
 		// Drop the superblock reference to commit
 		write_ref begin(block_address superblock, validator v);
 		write_ref new_block(validator v);

src/version.rs.in

@@ -1,2 +0,0 @@
-pub const TOOLS_VERSION: &str = @THIN_PROVISIONING_TOOLS_VERSION@;
-

tests/cache_check.rs

@@ -0,0 +1,157 @@
+use anyhow::Result;
+
+mod common;
+
+use common::cache::*;
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "cache_check 0.9.0
+
+USAGE:
+    cache_check [FLAGS] <INPUT>
+
+FLAGS:
+        --auto-repair                Auto repair trivial issues.
+        --ignore-non-fatal-errors    Only return a non-zero exit code if a fatal error is found.
+    -q, --quiet                      Suppress output messages, return only exit code.
+        --super-block-only           Only check the superblock.
+        --skip-discards              Don't check the discard bitset
+        --skip-hints                 Don't check the hint array
+    -h, --help                       Prints help information
+    -V, --version                    Prints version information
+
+ARGS:
+    <INPUT>    Specify the input device to check";
+
+//------------------------------------------
+
+struct CacheCheck;
+
+impl<'a> Program<'a> for CacheCheck {
+    fn name() -> &'a str {
+        "cache_check"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        cache_check_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for CacheCheck {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+impl<'a> MetadataReader<'a> for CacheCheck {}
+
+//------------------------------------------
+
+test_accepts_help!(CacheCheck);
+test_accepts_version!(CacheCheck);
+test_rejects_bad_option!(CacheCheck);
+
+test_missing_input_arg!(CacheCheck);
+test_input_file_not_found!(CacheCheck);
+test_input_cannot_be_a_directory!(CacheCheck);
+test_unreadable_input_file!(CacheCheck);
+
+test_help_message_for_tiny_input_file!(CacheCheck);
+test_spot_xml_data!(CacheCheck);
+test_corrupted_input_data!(CacheCheck);
+
+//------------------------------------------
+
+#[test]
+fn failing_q() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_zeroed_md(&mut td)?;
+    let output = run_fail_raw(cache_check_cmd(args!["-q", &md]))?;
+    assert_eq!(output.stdout.len(), 0);
+    eprintln!(
+        "stderr = '{}'",
+        std::str::from_utf8(&output.stderr).unwrap()
+    );
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn failing_quiet() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_zeroed_md(&mut td)?;
+    let output = run_fail_raw(cache_check_cmd(args!["--quiet", &md]))?;
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn valid_metadata_passes() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    run_ok(cache_check_cmd(args![&md]))?;
+    Ok(())
+}
+
+// FIXME: put back in, I don't want to add the --debug- arg to the
+// tool again, so we should have a little library function for tweaking
+// metadata version.
+
+/*
+#[test]
+fn bad_metadata_version() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+    run_ok(
+        cache_restore_cmd(
+        args![
+            "-i",
+            &xml,
+            "-o",
+            &md,
+            "--debug-override-metadata-version",
+            "12345"
+        ],
+    ))?;
+    run_fail(cache_check_cmd(args![&md]))?;
+    Ok(())
+}
+*/

tests/cache_dump.rs

@@ -0,0 +1,119 @@
+use anyhow::Result;
+use std::fs::OpenOptions;
+use std::io::Write;
+
+mod common;
+
+use common::cache::*;
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "cache_dump 0.9.0
+Dump the cache metadata to stdout in XML format
+
+USAGE:
+    cache_dump [FLAGS] [OPTIONS] <INPUT>
+
+FLAGS:
+    -r, --repair     Repair the metadata whilst dumping it
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+    -o, --output <FILE>    Specify the output file rather than stdout
+
+ARGS:
+    <INPUT>    Specify the input device to dump";
+
+//------------------------------------------
+
+struct CacheDump;
+
+impl<'a> Program<'a> for CacheDump {
+    fn name() -> &'a str {
+        "cache_dump"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        cache_dump_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for CacheDump {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(CacheDump);
+test_accepts_version!(CacheDump);
+test_rejects_bad_option!(CacheDump);
+
+test_missing_input_arg!(CacheDump);
+test_input_file_not_found!(CacheDump);
+test_input_cannot_be_a_directory!(CacheDump);
+test_unreadable_input_file!(CacheDump);
+
+//------------------------------------------
+
+// TODO: share with thin_dump
+#[test]
+fn dump_restore_cycle() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let output = run_ok_raw(cache_dump_cmd(args![&md]))?;
+
+    let xml = td.mk_path("meta.xml");
+    let mut file = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .open(&xml)?;
+    file.write_all(&output.stdout[0..])?;
+    drop(file);
+
+    let md2 = mk_zeroed_md(&mut td)?;
+    run_ok(cache_restore_cmd(args!["-i", &xml, "-o", &md2]))?;
+
+    let output2 = run_ok_raw(cache_dump_cmd(args![&md2]))?;
+    assert_eq!(output.stdout, output2.stdout);
+
+    Ok(())
+}

tests/cache_metadata_size.rs

@@ -0,0 +1,186 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+
+//------------------------------------------
+
+const USAGE: &str = "cache_metadata_size 0.9.0
+Estimate the size of the metadata device needed for a given configuration.
+
+USAGE:
+    cache_metadata_size [OPTIONS] <--device-size <SECTORS> --block-size <SECTORS> | --nr-blocks <NUM>>
+
+FLAGS:
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+        --block-size <SECTORS>      Specify the size of each cache block
+        --device-size <SECTORS>     Specify total size of the fast device used in the cache
+        --max-hint-width <BYTES>    Specity the per-block hint width [default: 4]
+        --nr-blocks <NUM>           Specify the number of cache blocks";
+
+//------------------------------------------
+
+struct CacheMetadataSize;
+
+impl<'a> Program<'a> for CacheMetadataSize {
+    fn name() -> &'a str {
+        "cache_metadata_size"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        cache_metadata_size_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(CacheMetadataSize);
+test_accepts_version!(CacheMetadataSize);
+test_rejects_bad_option!(CacheMetadataSize);
+
+//------------------------------------------
+
+#[test]
+fn no_args() -> Result<()> {
+    let _stderr = run_fail(cache_metadata_size_cmd([""; 0]))?;
+    Ok(())
+}
+
+#[test]
+fn device_size_only() -> Result<()> {
+    let _stderr = run_fail(cache_metadata_size_cmd(args!["--device-size", "204800"]))?;
+    Ok(())
+}
+
+#[test]
+fn block_size_only() -> Result<()> {
+    let _stderr = run_fail(cache_metadata_size_cmd(args!["--block-size", "128"]))?;
+    Ok(())
+}
+
+/*
+#[test]
+fn conradictory_info_fails() -> Result<()> {
+    let stderr = run_fail(cache_metadata_size_cmd(
+        args![
+            "--device-size",
+            "102400",
+            "--block-size",
+            "1000",
+            "--nr-blocks",
+            "6"
+        ],
+    ))?;
+    assert_eq!(stderr, "Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size.");
+    Ok(())
+}
+
+#[test]
+fn all_args_agree() -> Result<()> {
+    let out = run_ok_raw(cache_metadata_size_cmd(
+        args![
+            "--device-size",
+            "102400",
+            "--block-size",
+            "100",
+            "--nr-blocks",
+            "1024"
+        ],
+    ))?;
+    let stdout = std::str::from_utf8(&out.stdout[..])
+        .unwrap()
+        .trim_end_matches(|c| c == '\n' || c == '\r')
+        .to_string();
+    assert_eq!(stdout, "8248 sectors");
+    assert_eq!(out.stderr.len(), 0);
+    Ok(())
+}
+*/
+
+#[test]
+fn dev_size_and_nr_blocks_conflicts() -> Result<()> {
+    run_fail(cache_metadata_size_cmd(args![
+        "--device-size",
+        "102400",
+        "--nr-blocks",
+        "1024"
+    ]))?;
+    Ok(())
+}
+
+#[test]
+fn block_size_and_nr_blocks_conflicts() -> Result<()> {
+    run_fail(cache_metadata_size_cmd(args![
+        "--block-size",
+        "100",
+        "--nr-blocks",
+        "1024"
+    ]))?;
+    Ok(())
+}
+
+#[test]
+fn nr_blocks_alone() -> Result<()> {
+    let out = run_ok_raw(cache_metadata_size_cmd(args!["--nr-blocks", "1024"]))?;
+    let stdout = std::str::from_utf8(&out.stdout[..])
+        .unwrap()
+        .trim_end_matches(|c| c == '\n' || c == '\r')
+        .to_string();
+    assert_eq!(stdout, "8248 sectors");
+    assert_eq!(out.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn dev_size_and_block_size_succeeds() -> Result<()> {
+    let out = run_ok_raw(cache_metadata_size_cmd(args![
+        "--device-size",
+        "102400",
+        "--block-size",
+        "100"
+    ]))?;
+    let stdout = std::str::from_utf8(&out.stdout[..])
+        .unwrap()
+        .trim_end_matches(|c| c == '\n' || c == '\r')
+        .to_string();
+    assert_eq!(stdout, "8248 sectors");
+    assert_eq!(out.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn large_nr_blocks() -> Result<()> {
+    let out = run_ok_raw(cache_metadata_size_cmd(args!["--nr-blocks", "67108864"]))?;
+    let stdout = std::str::from_utf8(&out.stdout[..])
+        .unwrap()
+        .trim_end_matches(|c| c == '\n' || c == '\r')
+        .to_string();
+    assert_eq!(stdout, "3678208 sectors");
+    assert_eq!(out.stderr.len(), 0);
+    Ok(())
+}
+
+//------------------------------------------

tests/cache_repair.rs

@@ -0,0 +1,102 @@
+use anyhow::Result;
+
+mod common;
+
+use common::cache::*;
+use common::common_args::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "cache_repair 0.9.0
+Repair binary cache metadata, and write it to a different device or file
+
+USAGE:
+    cache_repair [FLAGS] --input <FILE> --output <FILE>
+
+FLAGS:
+    -q, --quiet      Suppress output messages, return only exit code.
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+    -i, --input <FILE>     Specify the input device
+    -o, --output <FILE>    Specify the output device";
+
+//-----------------------------------------
+
+struct CacheRepair;
+
+impl<'a> Program<'a> for CacheRepair {
+    fn name() -> &'a str {
+        "cache_repair"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        cache_repair_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for CacheRepair {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        "bad checksum in superblock"
+    }
+}
+
+impl<'a> OutputProgram<'a> for CacheRepair {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+impl<'a> MetadataWriter<'a> for CacheRepair {
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+}
+
+//-----------------------------------------
+
+test_accepts_help!(CacheRepair);
+test_accepts_version!(CacheRepair);
+test_rejects_bad_option!(CacheRepair);
+
+test_input_file_not_found!(CacheRepair);
+test_input_cannot_be_a_directory!(CacheRepair);
+test_corrupted_input_data!(CacheRepair);
+
+test_missing_output_option!(CacheRepair);
+
+//-----------------------------------------

tests/cache_restore.rs

@@ -0,0 +1,180 @@
+use anyhow::Result;
+
+mod common;
+
+use common::cache::*;
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "cache_restore 0.9.0
+Convert XML format metadata to binary.
+
+USAGE:
+    cache_restore [FLAGS] --input <FILE> --output <FILE>
+
+FLAGS:
+    -q, --quiet      Suppress output messages, return only exit code.
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+    -i, --input <FILE>     Specify the input xml
+    -o, --output <FILE>    Specify the output device to check";
+
+//------------------------------------------
+
+struct CacheRestore;
+
+impl<'a> Program<'a> for CacheRestore {
+    fn name() -> &'a str {
+        "thin_restore"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        cache_restore_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for CacheRestore {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_xml(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        "" // we don't intent to verify error messages of XML parsing
+    }
+}
+
+impl<'a> OutputProgram<'a> for CacheRestore {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+impl<'a> MetadataWriter<'a> for CacheRestore {
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+}
+
+//-----------------------------------------
+
+test_accepts_help!(CacheRestore);
+test_accepts_version!(CacheRestore);
+
+test_missing_input_option!(CacheRestore);
+test_input_file_not_found!(CacheRestore);
+test_corrupted_input_data!(CacheRestore);
+
+test_missing_output_option!(CacheRestore);
+test_tiny_output_file!(CacheRestore);
+
+test_unwritable_output_file!(CacheRestore);
+
+//-----------------------------------------
+
+// TODO: share with thin_restore, era_restore
+
+fn quiet_flag(flag: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+
+    let output = run_ok_raw(cache_restore_cmd(args!["-i", &xml, "-o", &md, flag]))?;
+
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn accepts_q() -> Result<()> {
+    quiet_flag("-q")
+}
+
+#[test]
+fn accepts_quiet() -> Result<()> {
+    quiet_flag("--quiet")
+}
+
+//-----------------------------------------
+
+#[test]
+fn successfully_restores() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+    run_ok(cache_restore_cmd(args!["-i", &xml, "-o", &md]))?;
+    Ok(())
+}
+
+// FIXME: finish
+/*
+#[test]
+fn override_metadata_version() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+    run_ok(
+        cache_restore_cmd(
+        args![
+            "-i",
+            &xml,
+            "-o",
+            &md,
+            "--debug-override-metadata-version",
+            "10298"
+        ],
+    ))?;
+    Ok(())
+}
+*/
+
+// FIXME: finish
+/*
+#[test]
+fn accepts_omit_clean_shutdown() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+    run_ok(
+        cache_restore_cmd(
+        args!["-i", &xml, "-o", &md, "--omit-clean-shutdown"],
+    ))?;
+    Ok(())
+}
+*/
+
+//-----------------------------------------

tests/common/cache.rs

@@ -0,0 +1,35 @@
+use anyhow::Result;
+use std::path::PathBuf;
+
+use thinp::file_utils;
+//use thinp::io_engine::*;
+
+use crate::args;
+use crate::common::cache_xml_generator::{write_xml, CacheGen};
+use crate::common::process::*;
+use crate::common::target::*;
+use crate::common::test_dir::TestDir;
+
+//-----------------------------------------------
+
+pub fn mk_valid_xml(td: &mut TestDir) -> Result<PathBuf> {
+    let xml = td.mk_path("meta.xml");
+    let mut gen = CacheGen::new(512, 128, 1024, 80, 50); // bs, cblocks, oblocks, res, dirty
+    write_xml(&xml, &mut gen)?;
+    Ok(xml)
+}
+
+pub fn mk_valid_md(td: &mut TestDir) -> Result<PathBuf> {
+    let xml = td.mk_path("meta.xml");
+    let md = td.mk_path("meta.bin");
+
+    let mut gen = CacheGen::new(512, 4096, 32768, 80, 50);
+    write_xml(&xml, &mut gen)?;
+
+    let _file = file_utils::create_sized_file(&md, 4096 * 4096);
+    run_ok(cache_restore_cmd(args!["-i", &xml, "-o", &md]))?;
+
+    Ok(md)
+}
+
+//-----------------------------------------------

tests/common/cache_xml_generator.rs

@@ -0,0 +1,95 @@
+use anyhow::Result;
+use rand::prelude::*;
+use std::collections::HashSet;
+use std::fs::OpenOptions;
+use std::path::Path;
+use thinp::cache::ir::{self, MetadataVisitor};
+use thinp::cache::xml;
+
+//------------------------------------------
+
+pub trait XmlGen {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()>;
+}
+
+pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> {
+    let xml_out = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .open(path)?;
+    let mut w = xml::XmlWriter::new(xml_out);
+
+    g.generate_xml(&mut w)
+}
+
+pub struct CacheGen {
+    block_size: u32,
+    nr_cache_blocks: u32,
+    nr_origin_blocks: u64,
+    percent_resident: u8,
+    percent_dirty: u8,
+}
+
+impl CacheGen {
+    pub fn new(
+        block_size: u32,
+        nr_cache_blocks: u32,
+        nr_origin_blocks: u64,
+        percent_resident: u8,
+        percent_dirty: u8,
+    ) -> Self {
+        CacheGen {
+            block_size,
+            nr_cache_blocks,
+            nr_origin_blocks,
+            percent_resident,
+            percent_dirty,
+        }
+    }
+}
+
+impl XmlGen for CacheGen {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        v.superblock_b(&ir::Superblock {
+            uuid: "".to_string(),
+            block_size: self.block_size,
+            nr_cache_blocks: self.nr_cache_blocks,
+            policy: "smq".to_string(),
+            hint_width: 4,
+        })?;
+
+        let nr_resident = (self.nr_cache_blocks * self.percent_resident as u32) / 100u32;
+        let mut cblocks = (0..self.nr_cache_blocks).collect::<Vec<u32>>();
+        cblocks.shuffle(&mut rand::thread_rng());
+        cblocks.truncate(nr_resident as usize);
+        cblocks.sort_unstable();
+
+        v.mappings_b()?;
+        {
+            let mut used = HashSet::new();
+            let mut rng = rand::thread_rng();
+            for cblock in cblocks {
+                let mut oblock = 0u64;
+                while used.contains(&oblock) {
+                    oblock = rng.gen_range(0..self.nr_origin_blocks);
+                }
+
+                used.insert(oblock);
+                // FIXME: dirty should vary
+                v.mapping(&ir::Map {
+                    cblock,
+                    oblock,
+                    dirty: false,
+                })?;
+            }
+        }
+        v.mappings_e()?;
+
+        v.superblock_e()?;
+        Ok(())
+    }
+}
+
+//------------------------------------------

tests/common/common_args.rs

@@ -0,0 +1,103 @@
+use anyhow::Result;
+
+use thinp::version::tools_version;
+
+use crate::args;
+use crate::common::process::*;
+use crate::common::program::*;
+
+//------------------------------------------
+// help
+
+pub fn test_help_short<'a, P>() -> Result<()>
+where
+    P: Program<'a>,
+{
+    let stdout = run_ok(P::cmd(args!["-h"]))?;
+    assert_eq!(stdout, P::usage());
+    Ok(())
+}
+
+pub fn test_help_long<'a, P>() -> Result<()>
+where
+    P: Program<'a>,
+{
+    let stdout = run_ok(P::cmd(vec!["--help"]))?;
+    assert_eq!(stdout, P::usage());
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_accepts_help {
+    ($program: ident) => {
+        #[test]
+        fn accepts_h() -> Result<()> {
+            test_help_short::<$program>()
+        }
+
+        #[test]
+        fn accepts_help() -> Result<()> {
+            test_help_long::<$program>()
+        }
+    };
+}
+
+//------------------------------------------
+// version
+
+pub fn test_version_short<'a, P>() -> Result<()>
+where
+    P: Program<'a>,
+{
+    let stdout = run_ok(P::cmd(args!["-V"]))?;
+    assert!(stdout.contains(tools_version()));
+    Ok(())
+}
+
+pub fn test_version_long<'a, P>() -> Result<()>
+where
+    P: Program<'a>,
+{
+    let stdout = run_ok(P::cmd(args!["--version"]))?;
+    assert!(stdout.contains(tools_version()));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_accepts_version {
+    ($program: ident) => {
+        #[test]
+        fn accepts_v() -> Result<()> {
+            test_version_short::<$program>()
+        }
+
+        #[test]
+        fn accepts_version() -> Result<()> {
+            test_version_long::<$program>()
+        }
+    };
+}
+
+//------------------------------------------
+
+pub fn test_rejects_bad_option<'a, P>() -> Result<()>
+where
+    P: Program<'a>,
+{
+    let option = "--hedgehogs-only";
+    let stderr = run_fail(P::cmd(args![option]))?;
+    assert!(stderr.contains(&P::bad_option_hint(option)));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_rejects_bad_option {
+    ($program: ident) => {
+        #[test]
+        fn rejects_bad_option() -> Result<()> {
+            test_rejects_bad_option::<$program>()
+        }
+    };
+}
+
+//------------------------------------------

tests/common/era.rs

@@ -0,0 +1,34 @@
+use anyhow::Result;
+use std::path::PathBuf;
+
+use thinp::file_utils;
+
+use crate::args;
+use crate::common::era_xml_generator::{write_xml, CleanShutdownMeta};
+use crate::common::process::*;
+use crate::common::target::*;
+use crate::common::test_dir::TestDir;
+
+//-----------------------------------------------
+
+pub fn mk_valid_xml(td: &mut TestDir) -> Result<PathBuf> {
+    let xml = td.mk_path("meta.xml");
+    let mut gen = CleanShutdownMeta::new(128, 256, 32, 4); // bs, nr_blocks, era, nr_wsets
+    write_xml(&xml, &mut gen)?;
+    Ok(xml)
+}
+
+pub fn mk_valid_md(td: &mut TestDir) -> Result<PathBuf> {
+    let xml = td.mk_path("meta.xml");
+    let md = td.mk_path("meta.bin");
+
+    let mut gen = CleanShutdownMeta::new(128, 256, 32, 4);
+    write_xml(&xml, &mut gen)?;
+
+    let _file = file_utils::create_sized_file(&md, 4096 * 4096);
+    run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md]))?;
+
+    Ok(md)
+}
+
+//-----------------------------------------------

tests/common/era_xml_generator.rs

@@ -0,0 +1,157 @@
+use anyhow::Result;
+use rand::prelude::*;
+use std::fs::OpenOptions;
+use std::path::Path;
+use thinp::era::ir::{self, MetadataVisitor};
+use thinp::era::xml;
+
+//------------------------------------------
+
+pub trait XmlGen {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()>;
+}
+
+pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> {
+    let xml_out = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .open(path)?;
+    let mut w = xml::XmlWriter::new(xml_out, false);
+
+    g.generate_xml(&mut w)
+}
+
+//------------------------------------------
+
+// Ordered sequence generator where each element has an independent probability
+// of being present.
+struct IndependentSequence {
+    begin: u32,
+    end: u32,
+    prob: u32,
+    rng: ThreadRng,
+}
+
+impl IndependentSequence {
+    fn new(begin: u32, end: u32, prob: u32) -> IndependentSequence {
+        IndependentSequence {
+            begin,
+            end,
+            prob,
+            rng: rand::thread_rng(),
+        }
+    }
+}
+
+impl Iterator for IndependentSequence {
+    type Item = std::ops::Range<u32>;
+
+    // FIXME: reduce complexity
+    fn next(&mut self) -> Option<std::ops::Range<u32>> {
+        if self.begin >= self.end {
+            return None;
+        }
+
+        let mut b = self.begin;
+        while b < self.end && self.rng.gen_range(0..100) >= self.prob {
+            b += 1;
+        }
+
+        if b == self.end {
+            return None;
+        }
+
+        let mut e = b + 1;
+        while e < self.end && self.rng.gen_range(0..100) < self.prob {
+            e += 1;
+        }
+        self.begin = e + 1;
+
+        Some(std::ops::Range { start: b, end: e })
+    }
+}
+
+//------------------------------------------
+
+fn create_superblock(block_size: u32, nr_blocks: u32, current_era: u32) -> ir::Superblock {
+    ir::Superblock {
+        uuid: "".to_string(),
+        block_size,
+        nr_blocks,
+        current_era,
+    }
+}
+
+pub struct CleanShutdownMeta {
+    block_size: u32,
+    nr_blocks: u32,
+    current_era: u32,
+    nr_writesets: u32,
+}
+
+impl CleanShutdownMeta {
+    pub fn new(block_size: u32, nr_blocks: u32, current_era: u32, nr_writesets: u32) -> Self {
+        CleanShutdownMeta {
+            block_size,
+            nr_blocks,
+            current_era,
+            nr_writesets,
+        }
+    }
+
+    fn generate_writeset(v: &mut dyn MetadataVisitor, ws: &ir::Writeset) -> Result<()> {
+        v.writeset_b(ws)?;
+        let gen = IndependentSequence::new(0, ws.nr_bits, 10);
+        for seq in gen {
+            v.writeset_blocks(&ir::MarkedBlocks {
+                begin: seq.start,
+                len: seq.end - seq.start,
+            })?;
+        }
+        v.writeset_e()?;
+
+        Ok(())
+    }
+
+    fn generate_era_array(v: &mut dyn MetadataVisitor, nr_blocks: u32, max_era: u32) -> Result<()> {
+        let mut rng = rand::thread_rng();
+        v.era_b()?;
+        for b in 0..nr_blocks {
+            let era = rng.gen_range(0..max_era);
+            v.era(&ir::Era { block: b, era })?;
+        }
+        v.era_e()?;
+
+        Ok(())
+    }
+}
+
+impl XmlGen for CleanShutdownMeta {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        v.superblock_b(&create_superblock(
+            self.block_size,
+            self.nr_blocks,
+            self.current_era,
+        ))?;
+
+        let era_low = self.current_era - self.nr_writesets + 1;
+        for era in era_low..self.current_era + 1 {
+            Self::generate_writeset(
+                v,
+                &ir::Writeset {
+                    era,
+                    nr_bits: self.nr_blocks,
+                },
+            )?;
+        }
+
+        Self::generate_era_array(v, self.nr_blocks, era_low)?;
+
+        v.superblock_e()?;
+        Ok(())
+    }
+}
+
+//------------------------------------------

tests/common/fixture.rs

@@ -0,0 +1,69 @@
+use anyhow::Result;
+use std::fs::OpenOptions;
+use std::io::{Read, Write};
+use std::path::PathBuf;
+
+use thinp::file_utils;
+
+use crate::common::test_dir::TestDir;
+
+//------------------------------------------
+
+pub fn mk_zeroed_md(td: &mut TestDir) -> Result<PathBuf> {
+    let md = td.mk_path("meta.bin");
+    eprintln!("path = {:?}", md);
+    let _file = file_utils::create_sized_file(&md, 1024 * 1024 * 16);
+    Ok(md)
+}
+
+pub fn damage_superblock(path: &PathBuf) -> Result<()> {
+    let mut output = OpenOptions::new().read(false).write(true).open(path)?;
+    let buf = [0u8; 512];
+    output.write_all(&buf)?;
+    Ok(())
+}
+
+//------------------------------------------
+
+pub fn md5(md: &PathBuf) -> Result<String> {
+    let output = duct::cmd!("md5sum", "-b", &md).stdout_capture().run()?;
+    let csum = std::str::from_utf8(&output.stdout[0..])?.to_string();
+    let csum = csum.split_ascii_whitespace().next().unwrap().to_string();
+    Ok(csum)
+}
+
+// This checksums the file before and after the thunk is run to
+// ensure it is unchanged.
+pub fn ensure_untouched<F>(p: &PathBuf, thunk: F) -> Result<()>
+where
+    F: Fn() -> Result<()>,
+{
+    let csum = md5(p)?;
+    thunk()?;
+    assert_eq!(csum, md5(p)?);
+    Ok(())
+}
+
+pub fn superblock_all_zeroes(path: &PathBuf) -> Result<bool> {
+    let mut input = OpenOptions::new().read(true).write(false).open(path)?;
+    let mut buf = vec![0; 4096];
+    input.read_exact(&mut buf[0..])?;
+    for b in buf {
+        if b != 0 {
+            return Ok(false);
+        }
+    }
+
+    Ok(true)
+}
+
+pub fn ensure_superblock_zeroed<F>(p: &PathBuf, thunk: F) -> Result<()>
+where
+    F: Fn() -> Result<()>,
+{
+    thunk()?;
+    assert!(superblock_all_zeroes(p)?);
+    Ok(())
+}
+
+//------------------------------------------

tests/common/input_arg.rs

@@ -0,0 +1,296 @@
+use anyhow::Result;
+use std::ffi::OsStr;
+
+use thinp::file_utils;
+
+use crate::args;
+use crate::common::fixture::*;
+use crate::common::process::*;
+use crate::common::program::*;
+use crate::common::test_dir::*;
+use crate::common::thin_xml_generator::{write_xml, FragmentedS};
+
+//------------------------------------------
+// wrappers
+
+type ArgsBuilder = fn(&mut TestDir, &OsStr, &dyn Fn(&[&OsStr]) -> Result<()>) -> Result<()>;
+
+fn with_output_md_untouched(
+    td: &mut TestDir,
+    input: &OsStr,
+    thunk: &dyn Fn(&[&OsStr]) -> Result<()>,
+) -> Result<()> {
+    let output = mk_zeroed_md(td)?;
+    ensure_untouched(&output, || {
+        let args = args!["-i", input, "-o", &output];
+        thunk(&args)
+    })
+}
+
+fn with_output_superblock_zeroed(
+    td: &mut TestDir,
+    input: &OsStr,
+    thunk: &dyn Fn(&[&OsStr]) -> Result<()>,
+) -> Result<()> {
+    let output = mk_zeroed_md(td)?;
+    ensure_superblock_zeroed(&output, || {
+        let args = args!["-i", input, "-o", &output];
+        thunk(&args)
+    })
+}
+
+fn input_arg_only(
+    _td: &mut TestDir,
+    input: &OsStr,
+    thunk: &dyn Fn(&[&OsStr]) -> Result<()>,
+) -> Result<()> {
+    let args = args![input];
+    thunk(&args)
+}
+
+fn build_args_fn(t: ArgType) -> Result<ArgsBuilder> {
+    match t {
+        ArgType::InputArg => Ok(input_arg_only),
+        ArgType::IoOptions => Ok(with_output_md_untouched),
+    }
+}
+
+//------------------------------------------
+// test invalid arguments
+
+pub fn test_missing_input_arg<'a, P>() -> Result<()>
+where
+    P: InputProgram<'a>,
+{
+    let args: [&str; 0] = [];
+    let stderr = run_fail(P::cmd(args))?;
+    assert!(stderr.contains(P::missing_input_arg()));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_missing_input_arg {
+    ($program: ident) => {
+        #[test]
+        fn missing_input_arg() -> Result<()> {
+            test_missing_input_arg::<$program>()
+        }
+    };
+}
+
+pub fn test_missing_input_option<'a, P>() -> Result<()>
+where
+    P: InputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+    let output = mk_zeroed_md(&mut td)?;
+    ensure_untouched(&output, || {
+        let stderr = run_fail(P::cmd(args!["-o", &output]))?;
+        assert!(stderr.contains(P::missing_input_arg()));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_missing_input_option {
+    ($program: ident) => {
+        #[test]
+        fn missing_input_option() -> Result<()> {
+            test_missing_input_option::<$program>()
+        }
+    };
+}
+
+pub fn test_input_file_not_found<'a, P>() -> Result<()>
+where
+    P: InputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+
+    let wrapper = build_args_fn(P::arg_type())?;
+    wrapper(&mut td, "no-such-file".as_ref(), &|args: &[&OsStr]| {
+        let stderr = run_fail(P::cmd(args))?;
+        assert!(stderr.contains(P::file_not_found()));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_input_file_not_found {
+    ($program: ident) => {
+        #[test]
+        fn input_file_not_found() -> Result<()> {
+            test_input_file_not_found::<$program>()
+        }
+    };
+}
+
+pub fn test_input_cannot_be_a_directory<'a, P>() -> Result<()>
+where
+    P: InputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+
+    let wrapper = build_args_fn(P::arg_type())?;
+    wrapper(&mut td, "/tmp".as_ref(), &|args: &[&OsStr]| {
+        let stderr = run_fail(P::cmd(args))?;
+        assert!(stderr.contains("Not a block device or regular file"));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_input_cannot_be_a_directory {
+    ($program: ident) => {
+        #[test]
+        fn input_cannot_be_a_directory() -> Result<()> {
+            test_input_cannot_be_a_directory::<$program>()
+        }
+    };
+}
+
+pub fn test_unreadable_input_file<'a, P>() -> Result<()>
+where
+    P: InputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+
+    // input an unreadable file
+    let input = P::mk_valid_input(&mut td)?;
+    duct::cmd!("chmod", "-r", &input).run()?;
+
+    let wrapper = build_args_fn(P::arg_type())?;
+    wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| {
+        let stderr = run_fail(P::cmd(args))?;
+        assert!(stderr.contains("Permission denied"));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_unreadable_input_file {
+    ($program: ident) => {
+        #[test]
+        fn unreadable_input_file() -> Result<()> {
+            test_unreadable_input_file::<$program>()
+        }
+    };
+}
+
+//------------------------------------------
+// test invalid content
+
+pub fn test_tiny_input_file<'a, P>() -> Result<()>
+where
+    P: MetadataReader<'a>,
+{
+    let mut td = TestDir::new()?;
+
+    let input = td.mk_path("meta.bin");
+    file_utils::create_sized_file(&input, 1024)?;
+
+    let wrapper = build_args_fn(P::arg_type())?;
+    wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| {
+        run_fail(P::cmd(args))?;
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_tiny_input_file {
+    ($program: ident) => {
+        #[test]
+        fn tiny_input_file() -> Result<()> {
+            test_tiny_input_file::<$program>()
+        }
+    };
+}
+
+pub fn test_help_message_for_tiny_input_file<'a, P>() -> Result<()>
+where
+    P: MetadataReader<'a>,
+{
+    let mut td = TestDir::new()?;
+
+    let input = td.mk_path("meta.bin");
+    file_utils::create_sized_file(&input, 1024)?;
+
+    let wrapper = build_args_fn(P::arg_type())?;
+    wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| {
+        let stderr = run_fail(P::cmd(args))?;
+        eprintln!("actual: {:?}", stderr);
+        assert!(stderr.contains("Metadata device/file too small.  Is this binary metadata?"));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_help_message_for_tiny_input_file {
+    ($program: ident) => {
+        #[test]
+        fn prints_help_message_for_tiny_input_file() -> Result<()> {
+            test_help_message_for_tiny_input_file::<$program>()
+        }
+    };
+}
+
+pub fn test_spot_xml_data<'a, P>() -> Result<()>
+where
+    P: MetadataReader<'a>,
+{
+    let mut td = TestDir::new()?;
+
+    // input a large xml file
+    let input = td.mk_path("meta.xml");
+    let mut gen = FragmentedS::new(4, 10240);
+    write_xml(&input, &mut gen)?;
+
+    let wrapper = build_args_fn(P::arg_type())?;
+    wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| {
+        let stderr = run_fail(P::cmd(args))?;
+        let msg =
+            "This looks like XML.  This tool only checks the binary metadata format.".to_string();
+        assert!(stderr.contains(&msg));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_spot_xml_data {
+    ($program: ident) => {
+        #[test]
+        fn spot_xml_data() -> Result<()> {
+            test_spot_xml_data::<$program>()
+        }
+    };
+}
+
+pub fn test_corrupted_input_data<'a, P>() -> Result<()>
+where
+    P: InputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+    let input = mk_zeroed_md(&mut td)?;
+
+    let wrapper = match P::arg_type() {
+        ArgType::InputArg => input_arg_only,
+        ArgType::IoOptions => with_output_superblock_zeroed,
+    };
+    wrapper(&mut td, input.as_ref(), &|args: &[&OsStr]| {
+        let stderr = run_fail(P::cmd(args))?;
+        assert!(stderr.contains(P::corrupted_input()));
+        Ok(())
+    })
+}
+
+#[macro_export]
+macro_rules! test_corrupted_input_data {
+    ($program: ident) => {
+        #[test]
+        fn corrupted_input_data() -> Result<()> {
+            test_corrupted_input_data::<$program>()
+        }
+    };
+}
+
+//------------------------------------------

tests/common/mod.rs

@@ -0,0 +1,18 @@
+// suppress all the false alarms by cargo test
+// https://github.com/rust-lang/rust/issues/46379
+#![allow(dead_code)]
+
+pub mod cache;
+pub mod cache_xml_generator;
+pub mod common_args;
+pub mod era;
+pub mod era_xml_generator;
+pub mod fixture;
+pub mod input_arg;
+pub mod output_option;
+pub mod process;
+pub mod program;
+pub mod target;
+pub mod test_dir;
+pub mod thin;
+pub mod thin_xml_generator;

tests/common/output_option.rs

@@ -0,0 +1,133 @@
+use anyhow::Result;
+
+use thinp::file_utils;
+
+use crate::args;
+use crate::common::process::*;
+use crate::common::program::*;
+use crate::common::test_dir::*;
+
+//-----------------------------------------
+// test invalid arguments
+
+pub fn test_missing_output_option<'a, P>() -> Result<()>
+where
+    P: OutputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+    let input = P::mk_valid_input(&mut td)?;
+    let stderr = run_fail(P::cmd(args!["-i", &input]))?;
+    assert!(stderr.contains(P::missing_output_arg()));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_missing_output_option {
+    ($program: ident) => {
+        #[test]
+        fn missing_output_option() -> Result<()> {
+            test_missing_output_option::<$program>()
+        }
+    };
+}
+
+pub fn test_output_file_not_found<'a, P>() -> Result<()>
+where
+    P: MetadataWriter<'a>,
+{
+    let mut td = TestDir::new()?;
+    let input = P::mk_valid_input(&mut td)?;
+    let cmd = P::cmd(args!["-i", &input, "-o", "no-such-file"]);
+    let stderr = run_fail(cmd)?;
+
+    assert!(stderr.contains(<P as MetadataWriter>::file_not_found()));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_output_file_not_found {
+    ($program: ident) => {
+        #[test]
+        fn output_file_not_found() -> Result<()> {
+            test_output_file_not_found::<$program>()
+        }
+    };
+}
+
+pub fn test_output_cannot_be_a_directory<'a, P>() -> Result<()>
+where
+    P: OutputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+    let input = P::mk_valid_input(&mut td)?;
+    let stderr = run_fail(P::cmd(args!["-i", &input, "-o", "/tmp"]))?;
+    assert!(stderr.contains("Not a block device or regular file"));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_output_cannot_be_a_directory {
+    ($program: ident) => {
+        #[test]
+        fn output_cannot_be_a_directory() -> Result<()> {
+            test_output_cannot_be_a_directory::<$program>()
+        }
+    };
+}
+
+pub fn test_unwritable_output_file<'a, P>() -> Result<()>
+where
+    P: OutputProgram<'a>,
+{
+    let mut td = TestDir::new()?;
+    let input = P::mk_valid_input(&mut td)?;
+
+    let output = td.mk_path("meta.bin");
+    let _file = file_utils::create_sized_file(&output, 4_194_304);
+    duct::cmd!("chmod", "-w", &output).run()?;
+
+    let stderr = run_fail(P::cmd(args!["-i", &input, "-o", &output]))?;
+    assert!(stderr.contains("Permission denied"));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_unwritable_output_file {
+    ($program: ident) => {
+        #[test]
+        fn unwritable_output_file() -> Result<()> {
+            test_unwritable_output_file::<$program>()
+        }
+    };
+}
+
+//----------------------------------------
+// test invalid content
+
+// currently thin/cache_restore only
+pub fn test_tiny_output_file<'a, P>() -> Result<()>
+where
+    P: MetadataWriter<'a>,
+{
+    let mut td = TestDir::new()?;
+    let input = P::mk_valid_input(&mut td)?;
+
+    let output = td.mk_path("meta.bin");
+    let _file = file_utils::create_sized_file(&output, 4096);
+
+    let stderr = run_fail(P::cmd(args!["-i", &input, "-o", &output]))?;
+    assert!(stderr.contains("Output file too small"));
+    Ok(())
+}
+
+#[macro_export]
+macro_rules! test_tiny_output_file {
+    ($program: ident) => {
+        #[test]
+        fn tiny_output_file() -> Result<()> {
+            test_tiny_output_file::<$program>()
+        }
+    };
+}
+
+//-----------------------------------------

tests/common/process.rs

@@ -0,0 +1,121 @@
+use anyhow::Result;
+use std::ffi::OsString;
+use std::fmt;
+use std::process;
+
+//------------------------------------------
+
+#[macro_export]
+macro_rules! args {
+    ( $( $arg: expr ),* ) => {
+        {
+            use std::ffi::OsStr;
+            let args = [$( OsStr::new($arg) ),*];
+            args
+        }
+    };
+}
+
+/// Holds a set of arguments for a shell command
+#[derive(Debug)]
+pub struct Command {
+    program: OsString,
+    args: Vec<OsString>,
+}
+
+#[macro_export]
+macro_rules! cmd {
+    ( $program:expr $(, $arg:expr )* $(,)? ) => {
+        {
+          //  use std::ffi::OsString;
+            let args: &[OsString] = &[$( Into::<OsString>::into($arg) ),*];
+            Command::new($program, args)
+        }
+    };
+}
+
+impl Command {
+    pub fn new(program: OsString, args: Vec<OsString>) -> Self {
+        Command { program, args }
+    }
+
+    fn to_expr(&self) -> duct::Expression {
+        duct::cmd(&self.program, &self.args)
+    }
+}
+
+impl fmt::Display for Command {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.program.clone().into_string().unwrap())?;
+        for a in &self.args {
+            write!(f, " {}", a.clone().into_string().unwrap())?;
+        }
+        Ok(())
+    }
+}
+
+fn log_output(output: &process::Output) {
+    use std::str::from_utf8;
+
+    if !output.stdout.is_empty() {
+        eprintln!("stdout: \n{}<<END>>", from_utf8(&output.stdout).unwrap());
+    }
+    if !output.stderr.is_empty() {
+        eprintln!("stderr: \n{}<<END>>", from_utf8(&output.stderr).unwrap());
+    }
+}
+
+// Returns stdout. The command must return zero.
+pub fn run_ok(command: Command) -> Result<String> {
+    eprintln!("run_ok: {}", command);
+
+    let command = command.to_expr().stdout_capture().stderr_capture();
+    let output = command.run()?;
+
+    log_output(&output);
+    assert!(output.status.success());
+
+    let stdout = std::str::from_utf8(&output.stdout[..])
+        .unwrap()
+        .trim_end_matches(|c| c == '\n' || c == '\r')
+        .to_string();
+
+    Ok(stdout)
+}
+
+// Returns the entire output. The command must return zero.
+pub fn run_ok_raw(command: Command) -> Result<std::process::Output> {
+    eprintln!("run_ok_raw: {}", command);
+    let command = command.to_expr().stdout_capture().stderr_capture();
+    let output = command.run()?;
+    log_output(&output);
+    assert!(output.status.success());
+    Ok(output)
+}
+
+// Returns stderr, a non zero status must be returned
+pub fn run_fail(command: Command) -> Result<String> {
+    eprintln!("run_fail: {}", command);
+    let command = command.to_expr().stdout_capture().stderr_capture();
+    let output = command.unchecked().run()?;
+    log_output(&output);
+    assert!(!output.status.success());
+    let stderr = std::str::from_utf8(&output.stderr[..])
+        .unwrap()
+        .trim_end_matches(|c| c == '\n' || c == '\r')
+        .to_string();
+
+    Ok(stderr)
+}
+
+// Returns the entire output, a non zero status must be returned
+pub fn run_fail_raw(command: Command) -> Result<std::process::Output> {
+    eprintln!("run_fail_raw: {}", command);
+    let command = command.to_expr().stdout_capture().stderr_capture();
+    let output = command.unchecked().run()?;
+    log_output(&output);
+    assert!(!output.status.success());
+    Ok(output)
+}
+
+//------------------------------------------

tests/common/program.rs

@@ -0,0 +1,52 @@
+use anyhow::Result;
+use std::path::PathBuf;
+
+pub use crate::common::process::*;
+use crate::common::test_dir::TestDir;
+
+//------------------------------------------
+
+pub enum ArgType {
+    InputArg,
+    IoOptions,
+}
+
+pub trait Program<'a> {
+    fn name() -> &'a str;
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>;
+    fn usage() -> &'a str;
+    fn arg_type() -> ArgType;
+
+    // error messages
+    fn bad_option_hint(option: &str) -> String;
+}
+
+pub trait InputProgram<'a>: Program<'a> {
+    fn mk_valid_input(td: &mut TestDir) -> Result<PathBuf>;
+
+    // error messages
+    fn missing_input_arg() -> &'a str;
+    fn file_not_found() -> &'a str;
+    fn corrupted_input() -> &'a str;
+}
+
+pub trait MetadataReader<'a>: InputProgram<'a> {}
+
+pub trait OutputProgram<'a>: InputProgram<'a> {
+    // error messages
+    fn missing_output_arg() -> &'a str;
+}
+
+// programs that write existed files
+pub trait MetadataWriter<'a>: OutputProgram<'a> {
+    // error messages
+    fn file_not_found() -> &'a str;
+}
+
+// programs that create output files (O_CREAT)
+pub trait MetadataCreator<'a>: OutputProgram<'a> {}
+
+//------------------------------------------

tests/common/target.rs

@@ -0,0 +1,215 @@
+use std::ffi::OsString;
+use std::path::PathBuf;
+
+use crate::common::process::*;
+
+//------------------------------------------
+
+pub fn cpp_cmd<S, I>(cmd: S, args: I) -> Command
+where
+    S: Into<OsString>,
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    let mut bin = PathBuf::from("bin");
+    bin.push(Into::<OsString>::into(cmd));
+
+    let mut args_ = Vec::new();
+    for a in args {
+        args_.push(Into::<OsString>::into(a));
+    }
+
+    Command::new(Into::<OsString>::into(bin.as_path()), args_)
+}
+
+pub fn rust_cmd<S, I>(cmd: S, args: I) -> Command
+where
+    S: Into<OsString>,
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    const RUST_PATH: &str = env!(concat!("CARGO_BIN_EXE_", "pdata_tools"));
+
+    let mut all_args = vec![Into::<OsString>::into(cmd)];
+    for a in args {
+        all_args.push(Into::<OsString>::into(a));
+    }
+
+    Command::new(Into::<OsString>::into(RUST_PATH), all_args)
+}
+
+pub fn thin_check_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("thin_check", args)
+}
+
+pub fn thin_rmap_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    cpp_cmd("thin_rmap", args)
+}
+
+pub fn thin_generate_metadata_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    cpp_cmd("thin_generate_metadata", args)
+}
+
+pub fn thin_generate_mappings_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    cpp_cmd("thin_generate_mappings", args)
+}
+
+pub fn thin_generate_damage_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    cpp_cmd("thin_generate_damage", args)
+}
+
+pub fn thin_restore_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    // rust_cmd("thin_restore", args)
+    cpp_cmd("thin_restore", args)
+}
+
+pub fn thin_repair_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("thin_restore", args)
+}
+
+pub fn thin_dump_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("thin_dump", args)
+}
+
+pub fn thin_delta_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    cpp_cmd("thin_delta", args)
+}
+
+pub fn thin_metadata_pack_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("thin_metadata_pack", args)
+}
+
+pub fn thin_metadata_unpack_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("thin_metadata_unpack", args)
+}
+
+pub fn cache_check_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("cache_check", args)
+}
+
+pub fn cache_dump_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("cache_dump", args)
+}
+
+pub fn cache_metadata_size_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("cache_metadata_size", args)
+}
+
+pub fn cache_restore_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("cache_restore", args)
+}
+
+pub fn cache_repair_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("cache_repair", args)
+}
+
+pub fn era_check_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("era_check", args)
+}
+
+pub fn era_dump_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("era_dump", args)
+}
+
+pub fn era_restore_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("era_restore", args)
+}
+
+pub fn era_repair_cmd<I>(args: I) -> Command
+where
+    I: IntoIterator,
+    I::Item: Into<OsString>,
+{
+    rust_cmd("era_repair", args)
+}
+
+//------------------------------------------
+
+pub mod msg {
+    pub const FILE_NOT_FOUND: &str = "Couldn't find input file";
+    pub const MISSING_INPUT_ARG: &str = "The following required arguments were not provided"; // TODO: be specific
+    pub const MISSING_OUTPUT_ARG: &str = "The following required arguments were not provided"; // TODO: be specific
+    pub const BAD_SUPERBLOCK: &str = "bad checksum in superblock";
+
+    pub fn bad_option_hint(option: &str) -> String {
+        format!("Found argument '{}' which wasn't expected", option)
+    }
+}
+
+//------------------------------------------

tests/common/test_dir.rs

@@ -0,0 +1,67 @@
+use anyhow::{anyhow, Result};
+use rand::prelude::*;
+use std::fs;
+use std::path::PathBuf;
+
+//---------------------------------------
+
+pub struct TestDir {
+    dir: PathBuf,
+    files: Vec<PathBuf>,
+    clean_up: bool,
+    file_count: usize,
+}
+
+fn mk_dir(prefix: &str) -> Result<PathBuf> {
+    for _n in 0..100 {
+        let mut p = PathBuf::new();
+        let nr = rand::thread_rng().gen_range(1000000..9999999);
+        p.push(format!("./{}_{}", prefix, nr));
+        if let Ok(()) = fs::create_dir(&p) {
+            return Ok(p);
+        }
+    }
+
+    Err(anyhow!("Couldn't create test directory"))
+}
+
+impl TestDir {
+    pub fn new() -> Result<TestDir> {
+        let dir = mk_dir("test_fixture")?;
+        Ok(TestDir {
+            dir,
+            files: Vec::new(),
+            clean_up: true,
+            file_count: 0,
+        })
+    }
+
+    pub fn dont_clean_up(&mut self) {
+        self.clean_up = false;
+    }
+
+    pub fn mk_path(&mut self, file: &str) -> PathBuf {
+        let mut p = PathBuf::new();
+        p.push(&self.dir);
+        p.push(PathBuf::from(format!("{:02}_{}", self.file_count, file)));
+        self.files.push(p.clone());
+        self.file_count += 1;
+        p
+    }
+}
+
+impl Drop for TestDir {
+    fn drop(&mut self) {
+        if self.clean_up {
+            while let Some(f) = self.files.pop() {
+                // It's not guaranteed that the path generated was actually created.
+                let _ignore = fs::remove_file(f);
+            }
+            fs::remove_dir(&self.dir).expect("couldn't remove test directory");
+        } else {
+            eprintln!("leaving test directory: {:?}", self.dir);
+        }
+    }
+}
+
+//---------------------------------------

tests/common/thin.rs

@@ -0,0 +1,126 @@
+use anyhow::Result;
+use std::path::PathBuf;
+
+use thinp::file_utils;
+use thinp::io_engine::*;
+
+use crate::args;
+use crate::common::fixture::*;
+use crate::common::process::*;
+use crate::common::target::*;
+use crate::common::test_dir::TestDir;
+use crate::common::thin_xml_generator::{write_xml, SingleThinS};
+
+//-----------------------------------------------
+
+pub fn mk_valid_xml(td: &mut TestDir) -> Result<PathBuf> {
+    let xml = td.mk_path("meta.xml");
+    let mut gen = SingleThinS::new(0, 1024, 2048, 2048);
+    write_xml(&xml, &mut gen)?;
+    Ok(xml)
+}
+
+pub fn mk_valid_md(td: &mut TestDir) -> Result<PathBuf> {
+    let xml = td.mk_path("meta.xml");
+    let md = td.mk_path("meta.bin");
+
+    let mut gen = SingleThinS::new(0, 1024, 20480, 20480);
+    write_xml(&xml, &mut gen)?;
+
+    let _file = file_utils::create_sized_file(&md, 4096 * 4096);
+    run_ok(thin_restore_cmd(args!["-i", &xml, "-o", &md]))?;
+
+    Ok(md)
+}
+
+//-----------------------------------------------
+
+// FIXME: replace mk_valid_md with this?
+pub fn prep_metadata(td: &mut TestDir) -> Result<PathBuf> {
+    let md = mk_zeroed_md(td)?;
+    let args = args!["-o", &md, "--format", "--nr-data-blocks", "102400"];
+    run_ok(thin_generate_metadata_cmd(args))?;
+
+    // Create a 2GB device
+    let args = args!["-o", &md, "--create-thin", "1"];
+    run_ok(thin_generate_metadata_cmd(args))?;
+    let args = args![
+        "-o",
+        &md,
+        "--dev-id",
+        "1",
+        "--size",
+        "2097152",
+        "--rw=randwrite",
+        "--seq-nr=16"
+    ];
+    run_ok(thin_generate_mappings_cmd(args))?;
+
+    // Take a few snapshots.
+    let mut snap_id = 2;
+    for _i in 0..10 {
+        // take a snapshot
+        let snap_id_str = snap_id.to_string();
+        let args = args!["-o", &md, "--create-snap", &snap_id_str, "--origin", "1"];
+        run_ok(thin_generate_metadata_cmd(args))?;
+
+        // partially overwrite the origin (64MB)
+        let args = args![
+            "-o",
+            &md,
+            "--dev-id",
+            "1",
+            "--size",
+            "2097152",
+            "--io-size",
+            "131072",
+            "--rw=randwrite",
+            "--seq-nr=16"
+        ];
+        run_ok(thin_generate_mappings_cmd(args))?;
+        snap_id += 1;
+    }
+
+    Ok(md)
+}
+
+pub fn set_needs_check(md: &PathBuf) -> Result<()> {
+    let args = args!["-o", &md, "--set-needs-check"];
+    run_ok(thin_generate_metadata_cmd(args))?;
+    Ok(())
+}
+
+pub fn generate_metadata_leaks(
+    md: &PathBuf,
+    nr_blocks: u64,
+    expected: u32,
+    actual: u32,
+) -> Result<()> {
+    let nr_blocks_str = nr_blocks.to_string();
+    let expected_str = expected.to_string();
+    let actual_str = actual.to_string();
+    let args = args![
+        "-o",
+        &md,
+        "--create-metadata-leaks",
+        "--nr-blocks",
+        &nr_blocks_str,
+        "--expected",
+        &expected_str,
+        "--actual",
+        &actual_str
+    ];
+    run_ok(thin_generate_damage_cmd(args))?;
+
+    Ok(())
+}
+
+pub fn get_needs_check(md: &PathBuf) -> Result<bool> {
+    use thinp::thin::superblock::*;
+
+    let engine = SyncIoEngine::new(md, 1, false)?;
+    let sb = read_superblock(&engine, SUPERBLOCK_LOCATION)?;
+    Ok(sb.flags.needs_check)
+}
+
+//-----------------------------------------------

tests/common/thin_xml_generator.rs

@@ -0,0 +1,538 @@
+use anyhow::{anyhow, Result};
+use rand::prelude::*;
+use std::collections::VecDeque;
+use std::fs::OpenOptions;
+use std::ops::Range;
+use std::path::Path;
+use thinp::thin::ir::{self, MetadataVisitor};
+use thinp::thin::xml;
+
+//------------------------------------------
+
+pub trait XmlGen {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()>;
+}
+
+pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> {
+    let xml_out = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .open(path)?;
+    let mut w = xml::XmlWriter::new(xml_out);
+
+    g.generate_xml(&mut w)
+}
+
+fn common_sb(nr_blocks: u64) -> ir::Superblock {
+    ir::Superblock {
+        uuid: "".to_string(),
+        time: 0,
+        transaction: 1,
+        flags: None,
+        version: None,
+        data_block_size: 128,
+        nr_data_blocks: nr_blocks,
+        metadata_snap: None,
+    }
+}
+
+//------------------------------------------
+
+pub struct EmptyPoolS {}
+
+impl XmlGen for EmptyPoolS {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        v.superblock_b(&common_sb(1024))?;
+        v.superblock_e()?;
+        Ok(())
+    }
+}
+
+//------------------------------------------
+
+pub struct SingleThinS {
+    pub offset: u64,
+    pub len: u64,
+    pub old_nr_data_blocks: u64,
+    pub new_nr_data_blocks: u64,
+}
+
+impl SingleThinS {
+    pub fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self {
+        SingleThinS {
+            offset,
+            len,
+            old_nr_data_blocks,
+            new_nr_data_blocks,
+        }
+    }
+}
+
+impl XmlGen for SingleThinS {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
+        v.device_b(&ir::Device {
+            dev_id: 0,
+            mapped_blocks: self.len,
+            transaction: 0,
+            creation_time: 0,
+            snap_time: 0,
+        })?;
+        v.map(&ir::Map {
+            thin_begin: 0,
+            data_begin: self.offset,
+            time: 0,
+            len: self.len,
+        })?;
+        v.device_e()?;
+        v.superblock_e()?;
+        Ok(())
+    }
+}
+
+//------------------------------------------
+
+pub struct FragmentedS {
+    pub nr_thins: u32,
+    pub thin_size: u64,
+    pub old_nr_data_blocks: u64,
+    pub new_nr_data_blocks: u64,
+}
+
+impl FragmentedS {
+    pub fn new(nr_thins: u32, thin_size: u64) -> Self {
+        let old_size = (nr_thins as u64) * thin_size;
+        FragmentedS {
+            nr_thins,
+            thin_size,
+            old_nr_data_blocks: (nr_thins as u64) * thin_size,
+            new_nr_data_blocks: old_size * 3 / 4,
+        }
+    }
+}
+
+#[derive(Clone)]
+struct ThinRun {
+    thin_id: u32,
+    thin_begin: u64,
+    len: u64,
+}
+
+#[derive(Clone, Debug, Copy)]
+struct MappedRun {
+    thin_id: u32,
+    thin_begin: u64,
+    data_begin: u64,
+    len: u64,
+}
+
+fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range<u64>) -> Vec<ThinRun> {
+    let mut runs = Vec::new();
+    let mut b = 0u64;
+    while b < total_len {
+        let len = u64::min(
+            total_len - b,
+            thread_rng().gen_range(run_len.start..run_len.end),
+        );
+        runs.push(ThinRun {
+            thin_id,
+            thin_begin: b,
+            len,
+        });
+        b += len;
+    }
+    runs
+}
+
+impl XmlGen for FragmentedS {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        // Allocate each thin fully, in runs between 1 and 16.
+        let mut runs = Vec::new();
+        for thin in 0..self.nr_thins {
+            runs.append(&mut mk_runs(thin, self.thin_size, 1..17));
+        }
+
+        // Shuffle
+        runs.shuffle(&mut rand::thread_rng());
+
+        // map across the data
+        let mut maps = Vec::new();
+        let mut b = 0;
+        for r in &runs {
+            maps.push(MappedRun {
+                thin_id: r.thin_id,
+                thin_begin: r.thin_begin,
+                data_begin: b,
+                len: r.len,
+            });
+            b += r.len;
+        }
+
+        // drop half the mappings, which leaves us free runs
+        let mut dropped = Vec::new();
+        for (i, m) in maps.iter().enumerate() {
+            if i % 2 == 0 {
+                dropped.push(*m);
+            }
+        }
+
+        // Unshuffle.  This isn't strictly necc. but makes the xml
+        // more readable.
+        use std::cmp::Ordering;
+        maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) {
+            Ordering::Equal => l.thin_begin.cmp(&r.thin_begin),
+            o => o,
+        });
+
+        // write the xml
+        v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
+        for thin in 0..self.nr_thins {
+            v.device_b(&ir::Device {
+                dev_id: thin,
+                mapped_blocks: self.thin_size,
+                transaction: 0,
+                creation_time: 0,
+                snap_time: 0,
+            })?;
+
+            for m in &dropped {
+                if m.thin_id != thin {
+                    continue;
+                }
+
+                v.map(&ir::Map {
+                    thin_begin: m.thin_begin,
+                    data_begin: m.data_begin,
+                    time: 0,
+                    len: m.len,
+                })?;
+            }
+
+            v.device_e()?;
+        }
+        v.superblock_e()?;
+        Ok(())
+    }
+}
+
+//------------------------------------------
+
+struct Allocator {
+    runs: VecDeque<Range<u64>>,
+}
+
+impl Allocator {
+    fn new_shuffled(total_len: u64, run_len: Range<u64>) -> Allocator {
+        let mut runs = Vec::new();
+
+        let mut b = 0u64;
+        while b < total_len {
+            let len = u64::min(
+                total_len - b,
+                thread_rng().gen_range(run_len.start..run_len.end),
+            );
+            runs.push(b..(b + len));
+            b += len;
+        }
+
+        runs.shuffle(&mut thread_rng());
+        let runs: VecDeque<Range<u64>> = runs.iter().cloned().collect();
+        Allocator { runs }
+    }
+
+    #[allow(dead_code)]
+    fn is_empty(&self) -> bool {
+        self.runs.is_empty()
+    }
+
+    fn alloc(&mut self, len: u64) -> Result<Vec<Range<u64>>> {
+        let mut len = len;
+        let mut runs = Vec::new();
+
+        while len > 0 {
+            let r = self.runs.pop_front();
+
+            if r.is_none() {
+                return Err(anyhow!("could not allocate; out of space"));
+            }
+
+            let r = r.unwrap();
+            let rlen = r.end - r.start;
+            if len < rlen {
+                runs.push(r.start..(r.start + len));
+
+                // We need to push something back.
+                self.runs.push_front((r.start + len)..r.end);
+                len = 0;
+            } else {
+                runs.push(r.start..r.end);
+                len -= rlen;
+            }
+        }
+
+        Ok(runs)
+    }
+}
+
+// Having explicitly unmapped regions makes it easier to
+// apply snapshots.
+#[derive(Clone)]
+enum Run {
+    Mapped { data_begin: u64, len: u64 },
+    UnMapped { len: u64 },
+}
+
+impl Run {
+    #[allow(dead_code)]
+    fn len(&self) -> u64 {
+        match self {
+            Run::Mapped {
+                data_begin: _data_begin,
+                len,
+            } => *len,
+            Run::UnMapped { len } => *len,
+        }
+    }
+
+    fn split(&self, n: u64) -> (Option<Run>, Option<Run>) {
+        if n == 0 {
+            (None, Some(self.clone()))
+        } else if self.len() <= n {
+            (Some(self.clone()), None)
+        } else {
+            match self {
+                Run::Mapped { data_begin, len } => (
+                    Some(Run::Mapped {
+                        data_begin: *data_begin,
+                        len: n,
+                    }),
+                    Some(Run::Mapped {
+                        data_begin: data_begin + n,
+                        len: len - n,
+                    }),
+                ),
+                Run::UnMapped { len } => (
+                    Some(Run::UnMapped { len: n }),
+                    Some(Run::UnMapped { len: len - n }),
+                ),
+            }
+        }
+    }
+}
+
+#[derive(Clone)]
+struct ThinDev {
+    thin_id: u32,
+    dev_size: u64,
+    runs: Vec<Run>,
+}
+
+impl ThinDev {
+    fn emit(&self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        v.device_b(&ir::Device {
+            dev_id: self.thin_id,
+            mapped_blocks: self.dev_size,
+            transaction: 0,
+            creation_time: 0,
+            snap_time: 0,
+        })?;
+
+        let mut b = 0;
+        for r in &self.runs {
+            match r {
+                Run::Mapped { data_begin, len } => {
+                    v.map(&ir::Map {
+                        thin_begin: b,
+                        data_begin: *data_begin,
+                        time: 0,
+                        len: *len,
+                    })?;
+                    b += len;
+                }
+                Run::UnMapped { len } => {
+                    b += len;
+                }
+            }
+        }
+
+        v.device_e()?;
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+enum SnapRunType {
+    Same,
+    Diff,
+    Hole,
+}
+
+#[derive(Clone)]
+struct SnapRun(SnapRunType, u64);
+
+fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result<ThinDev> {
+    let mut runs = Vec::new();
+    let mut b = 0;
+    while b < total_len {
+        let len = u64::min(thread_rng().gen_range(16..64), total_len - b);
+        match thread_rng().gen_range(0..2) {
+            0 => {
+                for data in allocator.alloc(len)? {
+                    assert!(data.end >= data.start);
+                    runs.push(Run::Mapped {
+                        data_begin: data.start,
+                        len: data.end - data.start,
+                    });
+                }
+            }
+            1 => {
+                runs.push(Run::UnMapped { len });
+            }
+            _ => {
+                return Err(anyhow!("bad value returned from rng"));
+            }
+        };
+
+        b += len;
+    }
+
+    Ok(ThinDev {
+        thin_id,
+        dev_size: total_len,
+        runs,
+    })
+}
+
+fn mk_snap_mapping(
+    total_len: u64,
+    run_len: Range<u64>,
+    same_percent: usize,
+    diff_percent: usize,
+) -> Vec<SnapRun> {
+    let mut runs = Vec::new();
+
+    let mut b = 0u64;
+    while b < total_len {
+        let len = u64::min(
+            total_len - b,
+            thread_rng().gen_range(run_len.start..run_len.end),
+        );
+
+        let n = thread_rng().gen_range(0..100);
+
+        if n < same_percent {
+            runs.push(SnapRun(SnapRunType::Same, len));
+        } else if n < diff_percent {
+            runs.push(SnapRun(SnapRunType::Diff, len));
+        } else {
+            runs.push(SnapRun(SnapRunType::Hole, len));
+        }
+
+        b += len;
+    }
+
+    runs
+}
+
+fn split_runs(mut n: u64, runs: &[Run]) -> (Vec<Run>, Vec<Run>) {
+    let mut before = Vec::new();
+    let mut after = Vec::new();
+
+    for r in runs {
+        match r.split(n) {
+            (Some(lhs), None) => {
+                before.push(lhs);
+            }
+            (Some(lhs), Some(rhs)) => {
+                before.push(lhs);
+                after.push(rhs);
+            }
+            (None, Some(rhs)) => {
+                after.push(rhs);
+            }
+            (None, None) => {}
+        }
+        n -= r.len();
+    }
+
+    (before, after)
+}
+
+fn apply_snap_runs(
+    origin: &[Run],
+    snap: &[SnapRun],
+    allocator: &mut Allocator,
+) -> Result<Vec<Run>> {
+    let mut origin = origin.to_owned();
+    let mut runs = Vec::new();
+
+    for SnapRun(st, slen) in snap {
+        let (os, rest) = split_runs(*slen, &origin);
+        match st {
+            SnapRunType::Same => {
+                for o in os {
+                    runs.push(o);
+                }
+            }
+            SnapRunType::Diff => {
+                for data in allocator.alloc(*slen)? {
+                    runs.push(Run::Mapped {
+                        data_begin: data.start,
+                        len: data.end - data.start,
+                    });
+                }
+            }
+            SnapRunType::Hole => {
+                runs.push(Run::UnMapped { len: *slen });
+            }
+        }
+
+        origin = rest;
+    }
+
+    Ok(runs)
+}
+
+// Snapshots share mappings, not neccessarily the entire ranges.
+pub struct SnapS {
+    pub len: u64,
+    pub nr_snaps: u32,
+
+    // Snaps will differ from the origin by this percentage
+    pub percent_change: usize,
+    pub old_nr_data_blocks: u64,
+    pub new_nr_data_blocks: u64,
+}
+
+impl SnapS {
+    pub fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self {
+        let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100;
+        let old_nr_data_blocks = len + 3 * delta;
+        let new_nr_data_blocks = len + 2 * delta;
+
+        SnapS {
+            len,
+            nr_snaps,
+            percent_change,
+            old_nr_data_blocks,
+            new_nr_data_blocks,
+        }
+    }
+}
+
+impl XmlGen for SnapS {
+    fn generate_xml(&mut self, v: &mut dyn MetadataVisitor) -> Result<()> {
+        let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512);
+        let origin = mk_origin(0, self.len, &mut allocator)?;
+
+        v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
+        origin.emit(v)?;
+        v.superblock_e()?;
+
+        Ok(())
+    }
+}
+
+//------------------------------------------

tests/era_check.rs

@@ -0,0 +1,118 @@
+use anyhow::Result;
+
+mod common;
+
+use common::cache::*;
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "era_check 0.9.0
+
+USAGE:
+    era_check [FLAGS] <INPUT>
+
+FLAGS:
+        --ignore-non-fatal-errors    Only return a non-zero exit code if a fatal error is found.
+    -q, --quiet                      Suppress output messages, return only exit code.
+        --super-block-only           Only check the superblock.
+    -h, --help                       Prints help information
+    -V, --version                    Prints version information
+
+ARGS:
+    <INPUT>    Specify the input device to check";
+
+//------------------------------------------
+
+struct EraCheck;
+
+impl<'a> Program<'a> for EraCheck {
+    fn name() -> &'a str {
+        "era_check"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        era_check_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for EraCheck {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+impl<'a> MetadataReader<'a> for EraCheck {}
+
+//------------------------------------------
+
+test_accepts_help!(EraCheck);
+test_accepts_version!(EraCheck);
+test_rejects_bad_option!(EraCheck);
+
+test_missing_input_arg!(EraCheck);
+test_input_file_not_found!(EraCheck);
+test_input_cannot_be_a_directory!(EraCheck);
+test_unreadable_input_file!(EraCheck);
+
+test_help_message_for_tiny_input_file!(EraCheck);
+test_spot_xml_data!(EraCheck);
+test_corrupted_input_data!(EraCheck);
+
+//------------------------------------------
+
+#[test]
+fn failing_q() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_zeroed_md(&mut td)?;
+    let output = run_fail_raw(era_check_cmd(args!["-q", &md]))?;
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn failing_quiet() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_zeroed_md(&mut td)?;
+    let output = run_fail_raw(era_check_cmd(args!["--quiet", &md]))?;
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+//------------------------------------------

tests/era_dump.rs

@@ -0,0 +1,123 @@
+use anyhow::Result;
+use std::fs::OpenOptions;
+use std::io::Write;
+
+mod common;
+
+use common::common_args::*;
+use common::era::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "era_dump 0.9.0
+Dump the era metadata to stdout in XML format
+
+USAGE:
+    era_dump [FLAGS] [OPTIONS] <INPUT>
+
+FLAGS:
+        --logical    Fold any unprocessed write sets into the final era array
+    -r, --repair     Repair the metadata whilst dumping it
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+    -o, --output <FILE>    Specify the output file rather than stdout
+
+ARGS:
+    <INPUT>    Specify the input device to dump";
+
+//------------------------------------------
+
+struct EraDump;
+
+impl<'a> Program<'a> for EraDump {
+    fn name() -> &'a str {
+        "era_dump"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        era_dump_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for EraDump {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+impl<'a> MetadataReader<'a> for EraDump {}
+
+//------------------------------------------
+
+test_accepts_help!(EraDump);
+test_accepts_version!(EraDump);
+test_rejects_bad_option!(EraDump);
+
+test_missing_input_arg!(EraDump);
+test_input_file_not_found!(EraDump);
+test_input_cannot_be_a_directory!(EraDump);
+test_unreadable_input_file!(EraDump);
+test_tiny_input_file!(EraDump);
+
+//------------------------------------------
+
+// TODO: share with thin_dump
+#[test]
+fn dump_restore_cycle() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let output = run_ok_raw(era_dump_cmd(args![&md]))?;
+
+    let xml = td.mk_path("meta.xml");
+    let mut file = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .open(&xml)?;
+    file.write_all(&output.stdout[0..])?;
+    drop(file);
+
+    let md2 = mk_zeroed_md(&mut td)?;
+    run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md2]))?;
+
+    let output2 = run_ok_raw(era_dump_cmd(args![&md2]))?;
+    assert_eq!(output.stdout, output2.stdout);
+
+    Ok(())
+}

tests/era_restore.rs

@@ -0,0 +1,143 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::era::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+
+//------------------------------------------
+
+const USAGE: &str = "era_restore 0.9.0
+Convert XML format metadata to binary.
+
+USAGE:
+    era_restore [FLAGS] --input <FILE> --output <FILE>
+
+FLAGS:
+    -q, --quiet      Suppress output messages, return only exit code.
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+    -i, --input <FILE>     Specify the input xml
+    -o, --output <FILE>    Specify the output device to check";
+
+//------------------------------------------
+
+struct EraRestore;
+
+impl<'a> Program<'a> for EraRestore {
+    fn name() -> &'a str {
+        "era_restore"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        era_restore_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for EraRestore {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_xml(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        "" // we don't intent to verify error messages of XML parsing
+    }
+}
+
+impl<'a> OutputProgram<'a> for EraRestore {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+impl<'a> MetadataWriter<'a> for EraRestore {
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+}
+
+//-----------------------------------------
+
+test_accepts_help!(EraRestore);
+test_accepts_version!(EraRestore);
+
+test_missing_input_option!(EraRestore);
+test_input_file_not_found!(EraRestore);
+test_corrupted_input_data!(EraRestore);
+
+test_missing_output_option!(EraRestore);
+test_tiny_output_file!(EraRestore);
+
+test_unwritable_output_file!(EraRestore);
+
+//-----------------------------------------
+
+// TODO: share with thin_restore, era_restore
+
+fn quiet_flag(flag: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+
+    let output = run_ok_raw(era_restore_cmd(args!["-i", &xml, "-o", &md, flag]))?;
+
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn accepts_q() -> Result<()> {
+    quiet_flag("-q")
+}
+
+#[test]
+fn accepts_quiet() -> Result<()> {
+    quiet_flag("--quiet")
+}
+
+//-----------------------------------------
+
+#[test]
+fn successfully_restores() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+    run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md]))?;
+    Ok(())
+}
+
+//-----------------------------------------

tests/thin_check.rs

@@ -0,0 +1,365 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = "thin_check 0.9.0
+Validates thin provisioning metadata on a device or file.
+
+USAGE:
+    thin_check [FLAGS] [OPTIONS] <INPUT>
+
+FLAGS:
+        --auto-repair                Auto repair trivial issues.
+        --clear-needs-check-flag     Clears the 'needs_check' flag in the superblock
+        --ignore-non-fatal-errors    Only return a non-zero exit code if a fatal error is found.
+    -m, --metadata-snapshot          Check the metadata snapshot on a live pool
+    -q, --quiet                      Suppress output messages, return only exit code.
+        --super-block-only           Only check the superblock.
+        --skip-mappings              Don't check the mapping tree
+    -h, --help                       Prints help information
+    -V, --version                    Prints version information
+
+OPTIONS:
+        --override-mapping-root <OVERRIDE_MAPPING_ROOT>    Specify a mapping root to use
+
+ARGS:
+    <INPUT>    Specify the input device to check";
+
+//-----------------------------------------
+
+struct ThinCheck;
+
+impl<'a> Program<'a> for ThinCheck {
+    fn name() -> &'a str {
+        "thin_check"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_check_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for ThinCheck {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+impl<'a> MetadataReader<'_> for ThinCheck {}
+
+//------------------------------------------
+
+test_accepts_help!(ThinCheck);
+test_accepts_version!(ThinCheck);
+test_rejects_bad_option!(ThinCheck);
+
+test_missing_input_arg!(ThinCheck);
+test_input_file_not_found!(ThinCheck);
+test_input_cannot_be_a_directory!(ThinCheck);
+test_unreadable_input_file!(ThinCheck);
+
+test_help_message_for_tiny_input_file!(ThinCheck);
+test_spot_xml_data!(ThinCheck);
+test_corrupted_input_data!(ThinCheck);
+
+//------------------------------------------
+// test exclusive flags
+
+fn accepts_flag(flag: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    run_ok(thin_check_cmd(args![flag, &md]))?;
+    Ok(())
+}
+
+#[test]
+fn accepts_superblock_only() -> Result<()> {
+    accepts_flag("--super-block-only")
+}
+
+#[test]
+fn accepts_skip_mappings() -> Result<()> {
+    accepts_flag("--skip-mappings")
+}
+
+#[test]
+fn accepts_ignore_non_fatal_errors() -> Result<()> {
+    accepts_flag("--ignore-non-fatal-errors")
+}
+
+#[test]
+fn accepts_clear_needs_check_flag() -> Result<()> {
+    accepts_flag("--clear-needs-check-flag")
+}
+
+#[test]
+fn accepts_auto_repair() -> Result<()> {
+    accepts_flag("--auto-repair")
+}
+
+//------------------------------------------
+// test the --quiet flag
+
+#[test]
+fn accepts_quiet() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+
+    let output = run_ok_raw(thin_check_cmd(args!["--quiet", &md]))?;
+    if !output.stdout.is_empty() {
+        eprintln!("stdout: {:?}", &std::str::from_utf8(&output.stdout));
+    }
+
+    if !output.stderr.is_empty() {
+        eprintln!("stderr: {:?}", &std::str::from_utf8(&output.stderr));
+    }
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+//------------------------------------------
+// test superblock-block-only
+
+#[test]
+fn detects_corrupt_superblock_with_superblock_only() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_zeroed_md(&mut td)?;
+    let _stderr = run_fail(thin_check_cmd(args!["--super-block-only", &md]))?;
+    Ok(())
+}
+
+//------------------------------------------
+// test info outputs
+
+#[test]
+fn prints_info_fields() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let stdout = run_ok(thin_check_cmd(args![&md]))?;
+    eprintln!("info: {:?}", stdout);
+    assert!(stdout.contains("TRANSACTION_ID="));
+    assert!(stdout.contains("METADATA_FREE_BLOCKS="));
+    Ok(())
+}
+
+//------------------------------------------
+// test compatibility between options
+
+#[test]
+fn auto_repair_incompatible_opts() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    run_fail(thin_check_cmd(args!["--auto-repair", "-m", &md]))?;
+    run_fail(thin_check_cmd(args![
+        "--auto-repair",
+        "--override-mapping-root",
+        "123",
+        &md
+    ]))?;
+    run_fail(thin_check_cmd(args![
+        "--auto-repair",
+        "--super-block-only",
+        &md
+    ]))?;
+    run_fail(thin_check_cmd(args![
+        "--auto-repair",
+        "--skip-mappings",
+        &md
+    ]))?;
+    run_fail(thin_check_cmd(args![
+        "--auto-repair",
+        "--ignore-non-fatal-errors",
+        &md
+    ]))?;
+    Ok(())
+}
+
+#[test]
+fn clear_needs_check_incompatible_opts() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    run_fail(thin_check_cmd(args!["--clear-needs-check-flag", "-m", &md]))?;
+    run_fail(thin_check_cmd(args![
+        "--clear-needs-check-flag",
+        "--override-mapping-root",
+        "123",
+        &md
+    ]))?;
+    run_fail(thin_check_cmd(args![
+        "--clear-needs-check-flag",
+        "--super-block-only",
+        &md
+    ]))?;
+    run_fail(thin_check_cmd(args![
+        "--clear-needs-check-flag",
+        "--ignore-non-fatal-errors",
+        &md
+    ]))?;
+    Ok(())
+}
+
+//------------------------------------------
+// test clear-needs-check
+
+#[test]
+fn clear_needs_check() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    set_needs_check(&md)?;
+
+    assert!(get_needs_check(&md)?);
+    run_ok(thin_check_cmd(args!["--clear-needs-check-flag", &md]))?;
+    assert!(!get_needs_check(&md)?);
+    Ok(())
+}
+
+#[test]
+fn no_clear_needs_check_if_error() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    set_needs_check(&md)?;
+    generate_metadata_leaks(&md, 1, 0, 1)?;
+    run_fail(thin_check_cmd(args!["--clear-needs-check-flag", &md]))?;
+    assert!(get_needs_check(&md)?);
+    Ok(())
+}
+
+#[test]
+fn clear_needs_check_if_skip_mappings() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    set_needs_check(&md)?;
+    generate_metadata_leaks(&md, 1, 0, 1)?;
+
+    assert!(get_needs_check(&md)?);
+    run_ok(thin_check_cmd(args![
+        "--clear-needs-check-flag",
+        "--skip-mappings",
+        &md
+    ]))?;
+    assert!(!get_needs_check(&md)?);
+    Ok(())
+}
+
+//------------------------------------------
+// test ignore-non-fatal-errors
+
+#[test]
+fn metadata_leaks_are_non_fatal() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+    generate_metadata_leaks(&md, 1, 0, 1)?;
+    run_fail(thin_check_cmd(args![&md]))?;
+    run_ok(thin_check_cmd(args!["--ignore-non-fatal-errors", &md]))?;
+    Ok(())
+}
+
+#[test]
+fn fatal_errors_cant_be_ignored() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    generate_metadata_leaks(&md, 1, 1, 0)?;
+    ensure_untouched(&md, || {
+        run_fail(thin_check_cmd(args!["--ignore-non-fatal-errors", &md]))?;
+        Ok(())
+    })
+}
+
+//------------------------------------------
+// test auto-repair
+
+#[test]
+fn auto_repair() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    eprintln!("here 0");
+
+    // auto-repair should have no effect on good metadata.
+    ensure_untouched(&md, || {
+        // run_ok(thin_check_cmd(args!["--auto-repair", &md]))?;
+        run_ok(thin_check_cmd(args![&md]))?;
+        Ok(())
+    })?;
+    eprintln!("here 0.5");
+
+    generate_metadata_leaks(&md, 16, 0, 1)?;
+    eprintln!("here 1");
+    run_fail(thin_check_cmd(args![&md]))?;
+    eprintln!("here 2");
+    run_ok(thin_check_cmd(args!["--auto-repair", &md]))?;
+    eprintln!("here 3");
+
+    run_ok(thin_check_cmd(args![&md]))?;
+    Ok(())
+}
+
+#[test]
+fn auto_repair_has_limits() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    generate_metadata_leaks(&md, 16, 1, 0)?;
+    ensure_untouched(&md, || {
+        run_fail(thin_check_cmd(args!["--auto-repair", &md]))?;
+        Ok(())
+    })?;
+    Ok(())
+}
+
+#[test]
+fn auto_repair_clears_needs_check() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = prep_metadata(&mut td)?;
+
+    set_needs_check(&md)?;
+    run_ok(thin_check_cmd(args!["--auto-repair", &md]))?;
+    assert!(!get_needs_check(&md)?);
+    Ok(())
+}
+
+//------------------------------------------

tests/thin_delta.rs

@@ -0,0 +1,87 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = "Usage: thin_delta [options] <device or file>\n\
+                     Options:\n  \
+                       {--thin1, --snap1, --root1}\n  \
+                       {--thin2, --snap2, --root2}\n  \
+                       {-m, --metadata-snap} [block#]\n  \
+                       {--verbose}\n  \
+                       {-h|--help}\n  \
+                       {-V|--version}";
+
+//------------------------------------------
+
+struct ThinDelta;
+
+impl<'a> Program<'a> for ThinDelta {
+    fn name() -> &'a str {
+        "thin_delta"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        cpp_cmd("thin_delta", args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        format!("unrecognized option '{}'", option)
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(ThinDelta);
+test_accepts_version!(ThinDelta);
+test_rejects_bad_option!(ThinDelta);
+
+//------------------------------------------
+
+#[test]
+fn snap1_unspecified() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let stderr = run_fail(thin_delta_cmd(args!["--snap2", "45", &md]))?;
+    assert!(stderr.contains("--snap1 or --root1 not specified"));
+    Ok(())
+}
+
+#[test]
+fn snap2_unspecified() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let stderr = run_fail(thin_delta_cmd(args!["--snap1", "45", &md]))?;
+    assert!(stderr.contains("--snap2 or --root2 not specified"));
+    Ok(())
+}
+
+#[test]
+fn dev_unspecified() -> Result<()> {
+    let stderr = run_fail(thin_delta_cmd(args!["--snap1", "45", "--snap2", "46"]))?;
+    // TODO: replace with msg::MISSING_INPUT_ARG once the rust version is ready
+    assert!(stderr.contains("No input file provided"));
+    Ok(())
+}
+
+//------------------------------------------

tests/thin_dump.rs

@@ -0,0 +1,244 @@
+use anyhow::Result;
+use std::fs::OpenOptions;
+use std::io::Write;
+
+mod common;
+
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = "thin_dump 0.9.0
+Dump thin-provisioning metadata to stdout in XML format
+
+USAGE:
+    thin_dump [FLAGS] [OPTIONS] <INPUT>
+
+FLAGS:
+    -q, --quiet            Suppress output messages, return only exit code.
+    -r, --repair           Repair the metadata whilst dumping it
+        --skip-mappings    Do not dump the mappings
+    -h, --help             Prints help information
+    -V, --version          Prints version information
+
+OPTIONS:
+        --data-block-size <SECTORS>                Provide the data block size for repairing
+    -m, --metadata-snapshot <METADATA_SNAPSHOT>    Access the metadata snapshot on a live pool
+        --nr-data-blocks <NUM>                     Override the number of data blocks if needed
+    -o, --output <FILE>                            Specify the output file rather than stdout
+        --transaction-id <NUM>                     Override the transaction id if needed
+
+ARGS:
+    <INPUT>    Specify the input device to dump";
+
+//-----------------------------------------
+
+struct ThinDump;
+
+impl<'a> Program<'a> for ThinDump {
+    fn name() -> &'a str {
+        "thin_dump"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_dump_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for ThinDump {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(ThinDump);
+test_accepts_version!(ThinDump);
+test_rejects_bad_option!(ThinDump);
+
+test_missing_input_arg!(ThinDump);
+test_input_file_not_found!(ThinDump);
+test_input_cannot_be_a_directory!(ThinDump);
+test_unreadable_input_file!(ThinDump);
+
+//------------------------------------------
+// test dump & restore cycle
+
+#[test]
+fn dump_restore_cycle() -> Result<()> {
+    let mut td = TestDir::new()?;
+
+    let md = mk_valid_md(&mut td)?;
+    let output = run_ok_raw(thin_dump_cmd(args![&md]))?;
+
+    let xml = td.mk_path("meta.xml");
+    let mut file = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .open(&xml)?;
+    file.write_all(&output.stdout[0..])?;
+    drop(file);
+
+    let md2 = mk_zeroed_md(&mut td)?;
+    run_ok(thin_restore_cmd(args!["-i", &xml, "-o", &md2]))?;
+
+    let output2 = run_ok_raw(thin_dump_cmd(args![&md2]))?;
+    assert_eq!(output.stdout, output2.stdout);
+
+    Ok(())
+}
+
+//------------------------------------------
+// test no stderr with a normal dump
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn no_stderr() -> Result<()> {
+    let mut td = TestDir::new()?;
+
+    let md = mk_valid_md(&mut td)?;
+    let output = run_ok_raw(thin_dump_cmd(args![&md]))?;
+
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+//------------------------------------------
+// test superblock overriding & repair
+// TODO: share with thin_repair
+
+fn override_something(flag: &str, value: &str, pattern: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let output = run_ok_raw(thin_dump_cmd(args![&md, flag, value]))?;
+
+    assert_eq!(output.stderr.len(), 0);
+    assert!(std::str::from_utf8(&output.stdout[0..])?.contains(pattern));
+    Ok(())
+}
+
+#[test]
+fn override_transaction_id() -> Result<()> {
+    override_something("--transaction-id", "2345", "transaction=\"2345\"")
+}
+
+#[test]
+fn override_data_block_size() -> Result<()> {
+    override_something("--data-block-size", "8192", "data_block_size=\"8192\"")
+}
+
+#[test]
+fn override_nr_data_blocks() -> Result<()> {
+    override_something("--nr-data-blocks", "234500", "nr_data_blocks=\"234500\"")
+}
+
+// FIXME: duplicate with superblock_succeeds in thin_repair.rs
+#[test]
+fn repair_superblock() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    let before = run_ok_raw(thin_dump_cmd(args![&md]))?;
+    damage_superblock(&md)?;
+
+    let after = run_ok_raw(thin_dump_cmd(args![
+        "--repair",
+        "--transaction-id=1",
+        "--data-block-size=128",
+        "--nr-data-blocks=20480",
+        &md
+    ]))?;
+    if !cfg!(feature = "rust_tests") {
+        assert_eq!(after.stderr.len(), 0);
+    }
+    assert_eq!(before.stdout, after.stdout);
+
+    Ok(())
+}
+
+//------------------------------------------
+// test compatibility between options
+// TODO: share with thin_repair
+
+#[test]
+fn missing_transaction_id() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    damage_superblock(&md)?;
+    let stderr = run_fail(thin_dump_cmd(args![
+        "--repair",
+        "--data-block-size=128",
+        "--nr-data-blocks=20480",
+        &md
+    ]))?;
+    assert!(stderr.contains("transaction id"));
+    Ok(())
+}
+
+#[test]
+fn missing_data_block_size() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    damage_superblock(&md)?;
+    let stderr = run_fail(thin_dump_cmd(args![
+        "--repair",
+        "--transaction-id=1",
+        "--nr-data-blocks=20480",
+        &md
+    ]))?;
+    assert!(stderr.contains("data block size"));
+    Ok(())
+}
+
+#[test]
+fn missing_nr_data_blocks() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    damage_superblock(&md)?;
+    let stderr = run_fail(thin_dump_cmd(args![
+        "--repair",
+        "--transaction-id=1",
+        "--data-block-size=128",
+        &md
+    ]))?;
+    assert!(stderr.contains("nr data blocks"));
+    Ok(())
+}
+
+//------------------------------------------

tests/thin_metadata_pack.rs

@@ -0,0 +1,96 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = concat!(
+    "thin_metadata_pack ",
+    include_str!("../VERSION"),
+    "Produces a compressed file of thin metadata.  Only packs metadata blocks that are actually used.\n\
+     \n\
+     USAGE:\n    \
+         thin_metadata_pack -i <DEV> -o <FILE>\n\
+     \n\
+     FLAGS:\n    \
+         -h, --help       Prints help information\n    \
+         -V, --version    Prints version information\n\
+     \n\
+     OPTIONS:\n    \
+         -i <DEV>         Specify thinp metadata binary device/file\n    \
+         -o <FILE>        Specify packed output file"
+);
+
+//------------------------------------------
+
+struct ThinMetadataPack;
+
+impl<'a> Program<'a> for ThinMetadataPack {
+    fn name() -> &'a str {
+        "thin_metadata_pack"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_metadata_pack_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for ThinMetadataPack {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        msg::BAD_SUPERBLOCK
+    }
+}
+
+impl<'a> OutputProgram<'a> for ThinMetadataPack {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(ThinMetadataPack);
+test_accepts_version!(ThinMetadataPack);
+test_rejects_bad_option!(ThinMetadataPack);
+
+test_missing_input_option!(ThinMetadataPack);
+test_missing_output_option!(ThinMetadataPack);
+test_input_file_not_found!(ThinMetadataPack);
+
+//-----------------------------------------

tests/thin_metadata_unpack.rs

@@ -0,0 +1,128 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = concat!(
+    "thin_metadata_unpack ",
+    include_str!("../VERSION"),
+    "Unpack a compressed file of thin metadata.\n\
+     \n\
+     USAGE:\n    \
+         thin_metadata_unpack -i <DEV> -o <FILE>\n\
+     \n\
+     FLAGS:\n    \
+         -h, --help       Prints help information\n    \
+         -V, --version    Prints version information\n\
+     \n\
+     OPTIONS:\n    \
+         -i <DEV>         Specify thinp metadata binary device/file\n    \
+         -o <FILE>        Specify packed output file"
+);
+
+//------------------------------------------
+
+struct ThinMetadataUnpack;
+
+impl<'a> Program<'a> for ThinMetadataUnpack {
+    fn name() -> &'a str {
+        "thin_metadata_pack"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_metadata_unpack_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for ThinMetadataUnpack {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_zeroed_md(td) // FIXME: make a real pack file
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        "Not a pack file"
+    }
+}
+
+impl<'a> OutputProgram<'a> for ThinMetadataUnpack {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(ThinMetadataUnpack);
+test_accepts_version!(ThinMetadataUnpack);
+test_rejects_bad_option!(ThinMetadataUnpack);
+
+test_missing_input_option!(ThinMetadataUnpack);
+test_input_file_not_found!(ThinMetadataUnpack);
+test_corrupted_input_data!(ThinMetadataUnpack);
+
+test_missing_output_option!(ThinMetadataUnpack);
+
+//------------------------------------------
+
+// TODO: share with thin_restore/cache_restore/era_restore
+
+#[test]
+fn end_to_end() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md_in = mk_valid_md(&mut td)?;
+    let md_out = mk_zeroed_md(&mut td)?;
+    run_ok(thin_metadata_pack_cmd(args![
+        "-i",
+        &md_in,
+        "-o",
+        "meta.pack"
+    ]))?;
+    run_ok(thin_metadata_unpack_cmd(args![
+        "-i",
+        "meta.pack",
+        "-o",
+        &md_out
+    ]))?;
+
+    let dump1 = run_ok(thin_dump_cmd(args![&md_in]))?;
+    let dump2 = run_ok(thin_dump_cmd(args![&md_out]))?;
+    assert_eq!(dump1, dump2);
+    Ok(())
+}
+
+//------------------------------------------

tests/thin_repair.rs

@@ -0,0 +1,224 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = "Usage: thin_repair [options] {device|file}\n\
+                     Options:\n  \
+                       {-h|--help}\n  \
+                       {-i|--input} <input metadata (binary format)>\n  \
+                       {-o|--output} <output metadata (binary format)>\n  \
+                       {--transaction-id} <natural>\n  \
+                       {--data-block-size} <natural>\n  \
+                       {--nr-data-blocks} <natural>\n  \
+                       {-V|--version}";
+
+//-----------------------------------------
+
+struct ThinRepair;
+
+impl<'a> Program<'a> for ThinRepair {
+    fn name() -> &'a str {
+        "thin_repair"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_repair_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for ThinRepair {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_md(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    #[cfg(not(feature = "rust_tests"))]
+    fn corrupted_input() -> &'a str {
+        "The following field needs to be provided on the command line due to corruption in the superblock"
+    }
+
+    #[cfg(feature = "rust_tests")]
+    fn corrupted_input() -> &'a str {
+        "data block size needs to be provided due to corruption in the superblock"
+    }
+}
+
+impl<'a> OutputProgram<'a> for ThinRepair {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+impl<'a> MetadataWriter<'a> for ThinRepair {
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+}
+
+//-----------------------------------------
+
+test_accepts_help!(ThinRepair);
+test_accepts_version!(ThinRepair);
+test_rejects_bad_option!(ThinRepair);
+
+test_input_file_not_found!(ThinRepair);
+test_input_cannot_be_a_directory!(ThinRepair);
+test_corrupted_input_data!(ThinRepair);
+
+test_missing_output_option!(ThinRepair);
+
+//-----------------------------------------
+// test output to a small file
+
+// TODO: share with thin_restore
+
+#[test]
+fn dont_repair_xml() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_zeroed_md(&mut td)?;
+    let xml = mk_valid_xml(&mut td)?;
+    run_fail(thin_repair_cmd(args!["-i", &xml, "-o", &md]))?;
+    Ok(())
+}
+
+//-----------------------------------------
+
+// TODO: share with thin_dump
+
+#[cfg(not(feature = "rust_tests"))]
+fn override_thing(flag: &str, val: &str, pattern: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md1 = mk_valid_md(&mut td)?;
+    let md2 = mk_zeroed_md(&mut td)?;
+    let output = run_ok_raw(thin_repair_cmd(args![flag, val, "-i", &md1, "-o", &md2]))?;
+    assert_eq!(output.stderr.len(), 0);
+    let output = run_ok(thin_dump_cmd(args![&md2]))?;
+    assert!(output.contains(pattern));
+    Ok(())
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn override_transaction_id() -> Result<()> {
+    override_thing("--transaction-id", "2345", "transaction=\"2345\"")
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn override_data_block_size() -> Result<()> {
+    override_thing("--data-block-size", "8192", "data_block_size=\"8192\"")
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn override_nr_data_blocks() -> Result<()> {
+    override_thing("--nr-data-blocks", "234500", "nr_data_blocks=\"234500\"")
+}
+
+// FIXME: that's repair_superblock in thin_dump.rs
+#[test]
+fn superblock_succeeds() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md1 = mk_valid_md(&mut td)?;
+    let original = run_ok_raw(thin_dump_cmd(args![&md1]))?;
+    if !cfg!(feature = "rust_tests") {
+        assert_eq!(original.stderr.len(), 0);
+    }
+    damage_superblock(&md1)?;
+    let md2 = mk_zeroed_md(&mut td)?;
+    run_ok(thin_repair_cmd(args![
+        "--transaction-id=1",
+        "--data-block-size=128",
+        "--nr-data-blocks=20480",
+        "-i",
+        &md1,
+        "-o",
+        &md2
+    ]))?;
+    let repaired = run_ok_raw(thin_dump_cmd(args![&md2]))?;
+    if !cfg!(feature = "rust_tests") {
+        assert_eq!(repaired.stderr.len(), 0);
+    }
+    assert_eq!(original.stdout, repaired.stdout);
+    Ok(())
+}
+
+//-----------------------------------------
+
+// TODO: share with thin_dump
+
+fn missing_thing(flag1: &str, flag2: &str, pattern: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md1 = mk_valid_md(&mut td)?;
+    damage_superblock(&md1)?;
+    let md2 = mk_zeroed_md(&mut td)?;
+    let stderr = run_fail(thin_repair_cmd(args![flag1, flag2, "-i", &md1, "-o", &md2]))?;
+    assert!(stderr.contains(pattern));
+    Ok(())
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn missing_transaction_id() -> Result<()> {
+    missing_thing(
+        "--data-block-size=128",
+        "--nr-data-blocks=20480",
+        "transaction id",
+    )
+}
+
+#[test]
+fn missing_data_block_size() -> Result<()> {
+    missing_thing(
+        "--transaction-id=1",
+        "--nr-data-blocks=20480",
+        "data block size",
+    )
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn missing_nr_data_blocks() -> Result<()> {
+    missing_thing(
+        "--transaction-id=1",
+        "--data-block-size=128",
+        "nr data blocks",
+    )
+}
+
+//-----------------------------------------

tests/thin_restore.rs

@@ -0,0 +1,162 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::fixture::*;
+use common::input_arg::*;
+use common::output_option::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = "Usage: thin_restore [options]\n\
+                     Options:\n  \
+                       {-h|--help}\n  \
+                       {-i|--input} <input xml file>\n  \
+                       {-o|--output} <output device or file>\n  \
+                       {--transaction-id} <natural>\n  \
+                       {--data-block-size} <natural>\n  \
+                       {--nr-data-blocks} <natural>\n  \
+                       {-q|--quiet}\n  \
+                       {-V|--version}";
+
+//------------------------------------------
+
+struct ThinRestore;
+
+impl<'a> Program<'a> for ThinRestore {
+    fn name() -> &'a str {
+        "thin_restore"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_restore_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::IoOptions
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+impl<'a> InputProgram<'a> for ThinRestore {
+    fn mk_valid_input(td: &mut TestDir) -> Result<std::path::PathBuf> {
+        mk_valid_xml(td)
+    }
+
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+
+    fn missing_input_arg() -> &'a str {
+        msg::MISSING_INPUT_ARG
+    }
+
+    fn corrupted_input() -> &'a str {
+        "" // we don't intent to verify error messages of XML parsing
+    }
+}
+
+impl<'a> OutputProgram<'a> for ThinRestore {
+    fn missing_output_arg() -> &'a str {
+        msg::MISSING_OUTPUT_ARG
+    }
+}
+
+impl<'a> MetadataWriter<'a> for ThinRestore {
+    fn file_not_found() -> &'a str {
+        msg::FILE_NOT_FOUND
+    }
+}
+
+//-----------------------------------------
+
+test_accepts_help!(ThinRestore);
+test_accepts_version!(ThinRestore);
+
+test_missing_input_option!(ThinRestore);
+test_input_file_not_found!(ThinRestore);
+test_corrupted_input_data!(ThinRestore);
+
+test_missing_output_option!(ThinRestore);
+test_tiny_output_file!(ThinRestore);
+
+test_unwritable_output_file!(ThinRestore);
+
+//-----------------------------------------
+
+// TODO: share with cache_restore, era_restore
+
+fn quiet_flag(flag: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+
+    let output = run_ok_raw(thin_restore_cmd(args!["-i", &xml, "-o", &md, flag]))?;
+
+    assert_eq!(output.stdout.len(), 0);
+    assert_eq!(output.stderr.len(), 0);
+    Ok(())
+}
+
+#[test]
+fn accepts_q() -> Result<()> {
+    quiet_flag("-q")
+}
+
+#[test]
+fn accepts_quiet() -> Result<()> {
+    quiet_flag("--quiet")
+}
+
+//-----------------------------------------
+
+// TODO: share with thin_dump
+#[cfg(not(feature = "rust_tests"))]
+fn override_something(flag: &str, value: &str, pattern: &str) -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    let md = mk_zeroed_md(&mut td)?;
+
+    run_ok(thin_restore_cmd(args!["-i", &xml, "-o", &md, flag, value]))?;
+
+    let output = run_ok(thin_dump_cmd(args![&md]))?;
+    assert!(output.contains(pattern));
+    Ok(())
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn override_transaction_id() -> Result<()> {
+    override_something("--transaction-id", "2345", "transaction=\"2345\"")
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn override_data_block_size() -> Result<()> {
+    override_something("--data-block-size", "8192", "data_block_size=\"8192\"")
+}
+
+#[test]
+#[cfg(not(feature = "rust_tests"))]
+fn override_nr_data_blocks() -> Result<()> {
+    override_something("--nr-data-blocks", "234500", "nr_data_blocks=\"234500\"")
+}
+
+//-----------------------------------------

tests/thin_rmap.rs

@@ -0,0 +1,107 @@
+use anyhow::Result;
+
+mod common;
+
+use common::common_args::*;
+use common::process::*;
+use common::program::*;
+use common::target::*;
+use common::test_dir::*;
+use common::thin::*;
+
+//------------------------------------------
+
+const USAGE: &str = "Usage: thin_rmap [options] {device|file}\n\
+                     Options:\n  \
+                       {-h|--help}\n  \
+                       {-V|--version}\n  \
+                       {--region <block range>}*\n\
+                     Where:\n  \
+                       <block range> is of the form <begin>..<one-past-the-end>\n  \
+                       for example 5..45 denotes blocks 5 to 44 inclusive, but not block 45";
+
+//------------------------------------------
+
+struct ThinRmap;
+
+impl<'a> Program<'a> for ThinRmap {
+    fn name() -> &'a str {
+        "thin_rmap"
+    }
+
+    fn cmd<I>(args: I) -> Command
+    where
+        I: IntoIterator,
+        I::Item: Into<std::ffi::OsString>,
+    {
+        thin_rmap_cmd(args)
+    }
+
+    fn usage() -> &'a str {
+        USAGE
+    }
+
+    fn arg_type() -> ArgType {
+        ArgType::InputArg
+    }
+
+    fn bad_option_hint(option: &str) -> String {
+        msg::bad_option_hint(option)
+    }
+}
+
+//------------------------------------------
+
+test_accepts_help!(ThinRmap);
+test_accepts_version!(ThinRmap);
+test_rejects_bad_option!(ThinRmap);
+
+//------------------------------------------
+
+#[test]
+fn valid_region_format_should_pass() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    run_ok(thin_rmap_cmd(args!["--region", "23..7890", &md]))?;
+    Ok(())
+}
+
+#[test]
+fn invalid_regions_should_fail() -> Result<()> {
+    let invalid_regions = [
+        "23,7890",
+        "23..six",
+        "found..7890",
+        "89..88",
+        "89..89",
+        "89..",
+        "",
+        "89...99",
+    ];
+    for r in &invalid_regions {
+        let mut td = TestDir::new()?;
+        let md = mk_valid_md(&mut td)?;
+        run_fail(thin_rmap_cmd(args![r, &md]))?;
+    }
+    Ok(())
+}
+
+#[test]
+fn multiple_regions_should_pass() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let md = mk_valid_md(&mut td)?;
+    run_ok(thin_rmap_cmd(args![
+        "--region", "1..23", "--region", "45..78", &md
+    ]))?;
+    Ok(())
+}
+
+#[test]
+fn junk_input() -> Result<()> {
+    let mut td = TestDir::new()?;
+    let xml = mk_valid_xml(&mut td)?;
+    run_fail(thin_rmap_cmd(args!["--region", "0..-1", &xml]))?;
+    Ok(())
+}
+
+//------------------------------------------

tests/thin_shrink.rs

@@ -1,15 +1,17 @@
 use anyhow::{anyhow, Result};
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 use rand::prelude::*;
-use std::collections::VecDeque;
 use std::fs::OpenOptions;
 use std::io::{Cursor, Read, Seek, SeekFrom, Write};
-use std::ops::Range;
-use std::path::{Path, PathBuf};
-use tempfile::tempdir;
+use std::path::Path;
 
 use thinp::file_utils;
-use thinp::thin::xml::{self, Visit};
+use thinp::thin::ir::{self, MetadataVisitor, Visit};
+use thinp::thin::xml;
+
+mod common;
+use common::test_dir::*;
+use common::thin_xml_generator::{write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen};
 
 //------------------------------------
 
@@ -90,8 +92,8 @@ struct ThinXmlVisitor<'a, V: ThinVisitor> {
     thin_id: Option<u32>,
 }
 
-impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> {
-    fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<Visit> {
+impl<'a, V: ThinVisitor> MetadataVisitor for ThinXmlVisitor<'a, V> {
+    fn superblock_b(&mut self, sb: &ir::Superblock) -> Result<Visit> {
         self.block_size = Some(sb.data_block_size);
         Ok(Visit::Continue)
     }
@@ -100,7 +102,15 @@ impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> {
         Ok(Visit::Continue)
     }
 
-    fn device_b(&mut self, d: &xml::Device) -> Result<Visit> {
+    fn def_shared_b(&mut self, _name: &str) -> Result<Visit> {
+        todo!();
+    }
+
+    fn def_shared_e(&mut self) -> Result<Visit> {
+        todo!();
+    }
+
+    fn device_b(&mut self, d: &ir::Device) -> Result<Visit> {
         self.thin_id = Some(d.dev_id);
         Ok(Visit::Continue)
     }
@@ -109,7 +119,7 @@ impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> {
         Ok(Visit::Continue)
     }
 
-    fn map(&mut self, m: &xml::Map) -> Result<Visit> {
+    fn map(&mut self, m: &ir::Map) -> Result<Visit> {
         for i in 0..m.len {
             let block = ThinBlock {
                 thin_id: self.thin_id.unwrap(),
@@ -122,6 +132,10 @@ impl<'a, V: ThinVisitor> xml::MetadataVisitor for ThinXmlVisitor<'a, V> {
         Ok(Visit::Continue)
     }
 
+    fn ref_shared(&mut self, _name: &str) -> Result<Visit> {
+        todo!();
+    }
+
     fn eof(&mut self) -> Result<Visit> {
         Ok(Visit::Stop)
     }
@@ -251,25 +265,6 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> {
 
 //------------------------------------
 
-fn mk_path(dir: &Path, file: &str) -> PathBuf {
-    let mut p = PathBuf::new();
-    p.push(dir);
-    p.push(PathBuf::from(file));
-    p
-}
-
-fn generate_xml(path: &Path, g: &mut dyn Scenario) -> Result<()> {
-    let xml_out = OpenOptions::new()
-        .read(false)
-        .write(true)
-        .create(true)
-        .truncate(true)
-        .open(path)?;
-    let mut w = xml::XmlWriter::new(xml_out);
-
-    g.generate_xml(&mut w)
-}
-
 fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> {
     let input = OpenOptions::new().read(true).write(false).open(xml_path)?;
 
@@ -304,17 +299,19 @@ fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
 }
 
 trait Scenario {
-    fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>;
     fn get_new_nr_blocks(&self) -> u64;
 }
 
-fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> {
-    let dir = tempdir()?;
-    let xml_before = mk_path(dir.path(), "before.xml");
-    let xml_after = mk_path(dir.path(), "after.xml");
-    let data_path = mk_path(dir.path(), "metadata.bin");
+fn test_shrink<S>(scenario: &mut S) -> Result<()>
+where
+    S: Scenario + XmlGen,
+{
+    let mut td = TestDir::new()?;
+    let xml_before = td.mk_path("before.xml");
+    let xml_after = td.mk_path("after.xml");
+    let data_path = td.mk_path("metadata.bin");
 
-    generate_xml(&xml_before, scenario)?;
+    write_xml(&xml_before, scenario)?;
     create_data_file(&data_path, &xml_before)?;
 
     let mut rng = rand::thread_rng();
@@ -332,28 +329,7 @@ fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> {
 
 //------------------------------------
 
-fn common_sb(nr_blocks: u64) -> xml::Superblock {
-    xml::Superblock {
-        uuid: "".to_string(),
-        time: 0,
-        transaction: 0,
-        flags: None,
-        version: None,
-        data_block_size: 32,
-        nr_data_blocks: nr_blocks,
-        metadata_snap: None,
-    }
-}
-
-struct EmptyPoolS {}
-
 impl Scenario for EmptyPoolS {
-    fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
-        v.superblock_b(&common_sb(1024))?;
-        v.superblock_e()?;
-        Ok(())
-    }
-
     fn get_new_nr_blocks(&self) -> u64 {
         512
     }
@@ -367,45 +343,7 @@ fn shrink_empty_pool() -> Result<()> {
 
 //------------------------------------
 
-struct SingleThinS {
-    offset: u64,
-    len: u64,
-    old_nr_data_blocks: u64,
-    new_nr_data_blocks: u64,
-}
-
-impl SingleThinS {
-    fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self {
-        SingleThinS {
-            offset,
-            len,
-            old_nr_data_blocks,
-            new_nr_data_blocks,
-        }
-    }
-}
-
 impl Scenario for SingleThinS {
-    fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
-        v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
-        v.device_b(&xml::Device {
-            dev_id: 0,
-            mapped_blocks: self.len,
-            transaction: 0,
-            creation_time: 0,
-            snap_time: 0,
-        })?;
-        v.map(&xml::Map {
-            thin_begin: 0,
-            data_begin: self.offset,
-            time: 0,
-            len: self.len,
-        })?;
-        v.device_e()?;
-        v.superblock_e()?;
-        Ok(())
-    }
-
     fn get_new_nr_blocks(&self) -> u64 {
         self.new_nr_data_blocks
     }
@@ -452,128 +390,7 @@ fn shrink_insufficient_space() -> Result<()> {
 
 //------------------------------------
 
-struct FragmentedS {
-    nr_thins: u32,
-    thin_size: u64,
-    old_nr_data_blocks: u64,
-    new_nr_data_blocks: u64,
-}
-
-impl FragmentedS {
-    fn new(nr_thins: u32, thin_size: u64) -> Self {
-        let old_size = (nr_thins as u64) * thin_size;
-        FragmentedS {
-            nr_thins,
-            thin_size,
-            old_nr_data_blocks: (nr_thins as u64) * thin_size,
-            new_nr_data_blocks: old_size * 3 / 4,
-        }
-    }
-}
-
-#[derive(Clone)]
-struct ThinRun {
-    thin_id: u32,
-    thin_begin: u64,
-    len: u64,
-}
-
-#[derive(Clone, Debug, Copy)]
-struct MappedRun {
-    thin_id: u32,
-    thin_begin: u64,
-    data_begin: u64,
-    len: u64,
-}
-
-fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range<u64>) -> Vec<ThinRun> {
-    let mut runs = Vec::new();
-    let mut b = 0u64;
-    while b < total_len {
-        let len = u64::min(
-            total_len - b,
-            thread_rng().gen_range(run_len.start, run_len.end),
-        );
-        runs.push(ThinRun {
-            thin_id: thin_id,
-            thin_begin: b,
-            len,
-        });
-        b += len;
-    }
-    runs
-}
-
 impl Scenario for FragmentedS {
-    fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
-        // Allocate each thin fully, in runs between 1 and 16.
-        let mut runs = Vec::new();
-        for thin in 0..self.nr_thins {
-            runs.append(&mut mk_runs(thin, self.thin_size, 1..17));
-        }
-
-        // Shuffle
-        runs.shuffle(&mut rand::thread_rng());
-
-        // map across the data
-        let mut maps = Vec::new();
-        let mut b = 0;
-        for r in &runs {
-            maps.push(MappedRun {
-                thin_id: r.thin_id,
-                thin_begin: r.thin_begin,
-                data_begin: b,
-                len: r.len,
-            });
-            b += r.len;
-        }
-
-        // drop half the mappings, which leaves us free runs
-        let mut dropped = Vec::new();
-        for i in 0..maps.len() {
-            if i % 2 == 0 {
-                dropped.push(maps[i].clone());
-            }
-        }
-
-        // Unshuffle.  This isn't strictly necc. but makes the xml
-        // more readable.
-        use std::cmp::Ordering;
-        maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) {
-            Ordering::Equal => l.thin_begin.cmp(&r.thin_begin),
-            o => o,
-        });
-
-        // write the xml
-        v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
-        for thin in 0..self.nr_thins {
-            v.device_b(&xml::Device {
-                dev_id: thin,
-                mapped_blocks: self.thin_size,
-                transaction: 0,
-                creation_time: 0,
-                snap_time: 0,
-            })?;
-
-            for m in &dropped {
-                if m.thin_id != thin {
-                    continue;
-                }
-
-                v.map(&xml::Map {
-                    thin_begin: m.thin_begin,
-                    data_begin: m.data_begin,
-                    time: 0,
-                    len: m.len,
-                })?;
-            }
-
-            v.device_e()?;
-        }
-        v.superblock_e()?;
-        Ok(())
-    }
-
     fn get_new_nr_blocks(&self) -> u64 {
         self.new_nr_data_blocks
     }
@@ -605,321 +422,7 @@ fn shrink_fragmented_thin_64() -> Result<()> {
 
 //------------------------------------
 
-struct Allocator {
-    runs: VecDeque<Range<u64>>,
-}
-
-impl Allocator {
-    fn new_shuffled(total_len: u64, run_len: Range<u64>) -> Allocator {
-        let mut runs = Vec::new();
-
-        let mut b = 0u64;
-        while b < total_len {
-            let len = u64::min(
-                total_len - b,
-                thread_rng().gen_range(run_len.start, run_len.end),
-            );
-            runs.push(b..(b + len));
-            b += len;
-        }
-
-        runs.shuffle(&mut thread_rng());
-        let runs: VecDeque<Range<u64>> = runs.iter().map(|r| r.clone()).collect();
-        Allocator { runs }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.runs.is_empty()
-    }
-
-    fn alloc(&mut self, len: u64) -> Result<Vec<Range<u64>>> {
-        let mut len = len;
-        let mut runs = Vec::new();
-
-        while len > 0 {
-            let r = self.runs.pop_front();
-
-            if r.is_none() {
-                return Err(anyhow!("could not allocate; out of space"));
-            }
-
-            let mut r = r.unwrap();
-            let rlen = r.end - r.start;
-            if len < rlen {
-                runs.push(r.start..(r.start + len));
-
-                // We need to push something back.
-                self.runs.push_front((r.start + len)..r.end);
-                len = 0;
-            } else {
-                runs.push(r.start..r.end);
-                len -= rlen;
-            }
-        }
-
-        Ok(runs)
-    }
-}
-
-// Having explicitly unmapped regions makes it easier to
-// apply snapshots.
-#[derive(Clone)]
-enum Run {
-    Mapped { data_begin: u64, len: u64 },
-    UnMapped { len: u64 },
-}
-
-impl Run {
-    fn len(&self) -> u64 {
-        match self {
-            Run::Mapped {
-                data_begin: _data_begin,
-                len,
-            } => *len,
-            Run::UnMapped { len } => *len,
-        }
-    }
-
-    fn split(&self, n: u64) -> (Option<Run>, Option<Run>) {
-        if n == 0 {
-            return (None, Some(self.clone()));
-        } else {
-            if self.len() <= n {
-                return (Some(self.clone()), None);
-            } else {
-                match self {
-                    Run::Mapped { data_begin, len } => (
-                        Some(Run::Mapped {
-                            data_begin: *data_begin,
-                            len: n,
-                        }),
-                        Some(Run::Mapped {
-                            data_begin: data_begin + n,
-                            len: len - n,
-                        }),
-                    ),
-                    Run::UnMapped { len } => (
-                        Some(Run::UnMapped { len: n }),
-                        Some(Run::UnMapped { len: len - n }),
-                    ),
-                }
-            }
-        }
-    }
-}
-
-#[derive(Clone)]
-struct ThinDev {
-    thin_id: u32,
-    dev_size: u64,
-    runs: Vec<Run>,
-}
-
-impl ThinDev {
-    fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
-        v.device_b(&xml::Device {
-            dev_id: self.thin_id,
-            mapped_blocks: self.dev_size,
-            transaction: 0,
-            creation_time: 0,
-            snap_time: 0,
-        })?;
-
-        let mut b = 0;
-        for r in &self.runs {
-            match r {
-                Run::Mapped { data_begin, len } => {
-                    v.map(&xml::Map {
-                        thin_begin: b,
-                        data_begin: *data_begin,
-                        time: 0,
-                        len: *len,
-                    })?;
-                    b += len;
-                }
-                Run::UnMapped { len } => {
-                    b += len;
-                }
-            }
-        }
-
-        v.device_e()?;
-        Ok(())
-    }
-}
-
-#[derive(Clone)]
-enum SnapRunType {
-    Same,
-    Diff,
-    Hole,
-}
-
-#[derive(Clone)]
-struct SnapRun(SnapRunType, u64);
-
-fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result<ThinDev> {
-    let mut runs = Vec::new();
-    let mut b = 0;
-    while b < total_len {
-        let len = u64::min(thread_rng().gen_range(16, 64), total_len - b);
-        match thread_rng().gen_range(0, 2) {
-            0 => {
-                for data in allocator.alloc(len)? {
-                    assert!(data.end >= data.start);
-                    runs.push(Run::Mapped {
-                        data_begin: data.start,
-                        len: data.end - data.start,
-                    });
-                }
-            }
-            1 => {
-                runs.push(Run::UnMapped { len });
-            }
-            _ => {
-                return Err(anyhow!("bad value returned from rng"));
-            }
-        };
-
-        b += len;
-    }
-
-    Ok(ThinDev {
-        thin_id,
-        dev_size: total_len,
-        runs,
-    })
-}
-
-fn mk_snap_mapping(
-    total_len: u64,
-    run_len: Range<u64>,
-    same_percent: usize,
-    diff_percent: usize,
-) -> Vec<SnapRun> {
-    let mut runs = Vec::new();
-
-    let mut b = 0u64;
-    while b < total_len {
-        let len = u64::min(
-            total_len - b,
-            thread_rng().gen_range(run_len.start, run_len.end),
-        );
-
-        let n = thread_rng().gen_range(0, 100);
-
-        if n < same_percent {
-            runs.push(SnapRun(SnapRunType::Same, len));
-        } else if n < diff_percent {
-            runs.push(SnapRun(SnapRunType::Diff, len));
-        } else {
-            runs.push(SnapRun(SnapRunType::Hole, len));
-        }
-
-        b += len;
-    }
-
-    runs
-}
-
-fn split_runs(mut n: u64, runs: &Vec<Run>) -> (Vec<Run>, Vec<Run>) {
-    let mut before = Vec::new();
-    let mut after = Vec::new();
-
-    for r in runs {
-        match r.split(n) {
-            (Some(lhs), None) => {
-                before.push(lhs);
-            }
-            (Some(lhs), Some(rhs)) => {
-                before.push(lhs);
-                after.push(rhs);
-            }
-            (None, Some(rhs)) => {
-                after.push(rhs);
-            }
-            (None, None) => {}
-        }
-        n -= r.len();
-    }
-
-    (before, after)
-}
-
-fn apply_snap_runs(
-    origin: &Vec<Run>,
-    snap: &Vec<SnapRun>,
-    allocator: &mut Allocator,
-) -> Result<Vec<Run>> {
-    let mut origin = origin.clone();
-    let mut runs = Vec::new();
-
-    for SnapRun(st, slen) in snap {
-        let (os, rest) = split_runs(*slen, &origin);
-        match st {
-            SnapRunType::Same => {
-                for o in os {
-                    runs.push(o);
-                }
-            }
-            SnapRunType::Diff => {
-                for data in allocator.alloc(*slen)? {
-                    runs.push(Run::Mapped {
-                        data_begin: data.start,
-                        len: data.end - data.start,
-                    });
-                }
-            }
-            SnapRunType::Hole => {
-                runs.push(Run::UnMapped { len: *slen });
-            }
-        }
-
-        origin = rest;
-    }
-
-    Ok(runs)
-}
-
-// Snapshots share mappings, not neccessarily the entire ranges.
-struct SnapS {
-    len: u64,
-    nr_snaps: u32,
-
-    // Snaps will differ from the origin by this percentage
-    percent_change: usize,
-    old_nr_data_blocks: u64,
-    new_nr_data_blocks: u64,
-}
-
-impl SnapS {
-    fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self {
-        let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100;
-        let old_nr_data_blocks = len + 3 * delta;
-        let new_nr_data_blocks = len + 2 * delta;
-
-        SnapS {
-            len,
-            nr_snaps,
-            percent_change,
-            old_nr_data_blocks,
-            new_nr_data_blocks,
-        }
-    }
-}
-
 impl Scenario for SnapS {
-    fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
-        let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512);
-        let origin = mk_origin(0, self.len, &mut allocator)?;
-
-        v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
-        origin.emit(v)?;
-        v.superblock_e()?;
-
-        Ok(())
-    }
-
     fn get_new_nr_blocks(&self) -> u64 {
         self.new_nr_data_blocks
     }

thin-provisioning/cache_stream.cc

@@ -62,7 +62,7 @@ chunk const &
 cache_stream::get()
 {
 	chunk_wrapper *w = new chunk_wrapper(*this);
-	return w->c_;
+	return w->c_; // wrapper will get freed by the put method
 }
 
 void

thin-provisioning/commands.cc

@@ -17,18 +17,6 @@ thin_provisioning::register_thin_commands(base::application &app)
 	app.add_cmd(command::ptr(new thin_repair_cmd()));
 	app.add_cmd(command::ptr(new thin_rmap_cmd()));
 	app.add_cmd(command::ptr(new thin_trim_cmd()));
-
-#ifdef DEV_TOOLS
-	app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
-	app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
-	app.add_cmd(command::ptr(new thin_scan_cmd()));
-	app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
-	app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
-	app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
-	app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));
-	app.add_cmd(command::ptr(new thin_show_metadata_cmd()));
-	app.add_cmd(command::ptr(new thin_journal_cmd()));
-#endif
 }
 
 //----------------------------------------------------------------

thin-provisioning/commands.h

@@ -71,34 +71,11 @@ namespace thin_provisioning {
 		virtual int run(int argc, char **argv);
 	};
 
-#ifdef DEV_TOOLS
-	class thin_ll_dump_cmd : public base::command {
+	//------------------------------------------------------
+
+	class thin_debug_cmd : public base::command {
 	public:
-		thin_ll_dump_cmd();
-
-		virtual void usage(std::ostream &out) const;
-		virtual int run(int argc, char **argv);
-	};
-
-	class thin_ll_restore_cmd : public base::command {
-	public:
-		thin_ll_restore_cmd();
-
-		virtual void usage(std::ostream &out) const;
-		virtual int run(int argc, char **argv);
-	};
-
-	class thin_scan_cmd : public base::command {
-	public:
-		thin_scan_cmd();
-
-		virtual void usage(std::ostream &out) const;
-		virtual int run(int argc, char **argv);
-	};
-
-	class thin_show_duplicates_cmd : public base::command {
-	public:
-		thin_show_duplicates_cmd();
+		thin_debug_cmd();
 		virtual void usage(std::ostream &out) const;
 		virtual int run(int argc, char **argv);
 	};
@@ -124,6 +101,52 @@ namespace thin_provisioning {
 		virtual int run(int argc, char **argv);
 	};
 
+	class thin_journal_cmd : public base::command {
+	public:
+		thin_journal_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_ll_dump_cmd : public base::command {
+	public:
+		thin_ll_dump_cmd();
+
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_ll_restore_cmd : public base::command {
+	public:
+		thin_ll_restore_cmd();
+
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_patch_superblock_cmd : public base::command {
+	public:
+		thin_patch_superblock_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_scan_cmd : public base::command {
+	public:
+		thin_scan_cmd();
+
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+	class thin_show_duplicates_cmd : public base::command {
+	public:
+		thin_show_duplicates_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
+
 	class thin_show_metadata_cmd : public base::command {
 	public:
 		thin_show_metadata_cmd();
@@ -131,14 +154,6 @@ namespace thin_provisioning {
 		virtual int run(int argc, char **argv);
 	};
 
-	class thin_journal_cmd : public base::command {
-	public:
-		thin_journal_cmd();
-		virtual void usage(std::ostream &out) const;
-		virtual int run(int argc, char **argv);
-	};
-#endif
-
 	void register_thin_commands(base::application &app);
 }
 

thin-provisioning/damage_generator.cc

@@ -63,6 +63,14 @@ void damage_generator::create_metadata_leaks(block_address nr_leaks,
 	std::set<block_address> leaks;
 	find_blocks(md_->metadata_sm_, nr_leaks, expected, leaks);
 
+	block_counter bc(true);
+	md_->metadata_sm_->count_metadata(bc);
+	block_address nr_blocks = md_->metadata_sm_->get_nr_blocks();
+	for (block_address b = 0; b < nr_blocks; b++) {
+		if (bc.get_count(b))
+			md_->tm_->mark_shadowed(b);
+	}
+
 	for (auto const &b : leaks)
 		md_->metadata_sm_->set_count(b, actual);
 }

thin-provisioning/devel_commands.cc

@@ -0,0 +1,24 @@
+#include "thin-provisioning/commands.h"
+
+using namespace base;
+using namespace thin_provisioning;
+
+//----------------------------------------------------------------
+
+void
+thin_provisioning::register_thin_commands(base::application &app)
+{
+	app.add_cmd(command::ptr(new thin_debug_cmd()));
+	app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
+	app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
+	app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
+	app.add_cmd(command::ptr(new thin_journal_cmd()));
+	app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
+	app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
+	app.add_cmd(command::ptr(new thin_patch_superblock_cmd()));
+	app.add_cmd(command::ptr(new thin_scan_cmd()));
+	app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));
+	app.add_cmd(command::ptr(new thin_show_metadata_cmd()));
+}
+
+//----------------------------------------------------------------

thin-provisioning/device_tree.h

@@ -25,6 +25,14 @@ namespace thin_provisioning {
 			uint32_t snapshotted_time_;
 		};
 
+		inline bool operator==(device_details const& lhs, device_details const& rhs) {
+			return false; // device_details are not comparable
+		}
+
+		inline bool operator!=(device_details const& lhs, device_details const& rhs) {
+			return !(lhs == rhs);
+		}
+
 		struct device_details_traits {
 			typedef device_details_disk disk_type;
 			typedef device_details value_type;

thin-provisioning/mapping_tree.h

@@ -24,6 +24,14 @@ namespace thin_provisioning {
 			uint32_t time_;
 		};
 
+		inline bool operator==(block_time const& lhs, block_time const& rhs) {
+			return lhs.block_ == rhs.block_;
+		}
+
+		inline bool operator!=(block_time const& lhs, block_time const& rhs) {
+			return !(lhs == rhs);
+		}
+
 		class block_time_ref_counter {
 		public:
 			block_time_ref_counter(space_map::ptr sm);