Merge branch 'master' of github.com:jthornber/thin-provisioning-tools

persistent-data/block.h

@@ -52,7 +52,8 @@ namespace persistent_data {
 		block_manager(std::string const &path,
 			      block_address nr_blocks,
 			      unsigned max_concurrent_locks,
-			      mode m);
+			      mode m,
+			      bool excl = true);
 
 		class read_ref {
 		public:
@@ -134,7 +135,8 @@ namespace persistent_data {
 		bool is_locked(block_address b) const;
 
 	private:
-		int open_or_create_block_file(std::string const &path, off_t file_size, mode m);
+		int open_or_create_block_file(std::string const &path, off_t file_size,
+					      mode m, bool excl);
 		void check(block_address b) const;
 
 		int fd_;

persistent-data/block.tcc

@@ -46,14 +46,23 @@ namespace {
 	// to exception.h
 	void syscall_failed(char const *call) {
 		ostringstream out;
-		out << "syscall '" << call << "' failed: " << base::error_string(errno);;
+		out << "syscall '" << call << "' failed: " << base::error_string(errno);
+		throw runtime_error(out.str());
+	}
+
+	void syscall_failed(string const &call, string const &message)
+	{
+		ostringstream out;
+		out << "syscall '" << call << "' failed: " << base::error_string(errno) << "\n"
+		    << message;
 		throw runtime_error(out.str());
 	}
 
 	int open_file(string const &path, int flags) {
 		int fd = ::open(path.c_str(), OPEN_FLAGS | flags, DEFAULT_MODE);
 		if (fd < 0)
-			syscall_failed("open");
+			syscall_failed("open",
+				 "Note: you cannot run this tool with these options on live metadata.");
 
 		return fd;
 	}
@@ -80,7 +89,7 @@ namespace {
 			throw runtime_error(out.str());
 		}
 
-		int fd = open_file(path, O_CREAT | O_RDWR);
+		int fd = open_file(path, O_CREAT | O_EXCL | O_RDWR);
 
 		int r = ::ftruncate(fd, file_size);
 		if (r < 0)
@@ -89,14 +98,18 @@ namespace {
 		return fd;
 	}
 
-	int open_block_file(string const &path, off_t min_size, bool writeable) {
+	int open_block_file(string const &path, off_t min_size, bool writeable, bool excl = true) {
 		if (!file_exists(path)) {
 			ostringstream out;
 			out << __FUNCTION__ << ": file '" << path << "' doesn't exist";
 			throw runtime_error(out.str());
 		}
 
-		return open_file(path, writeable ? O_RDWR : O_RDONLY);
+		int flags = writeable ? O_RDWR : O_RDONLY;
+		if (excl)
+			flags |= O_EXCL;
+
+		return open_file(path, flags);
 	}
 };
 
@@ -208,8 +221,9 @@ namespace persistent_data {
 	block_manager<BlockSize>::block_manager(std::string const &path,
 						block_address nr_blocks,
 						unsigned max_concurrent_blocks,
-						mode m)
-		: fd_(open_or_create_block_file(path, nr_blocks * BlockSize, m)),
+						mode m,
+						bool excl)
+		: fd_(open_or_create_block_file(path, nr_blocks * BlockSize, m, excl)),
 		  bc_(fd_, BlockSize >> SECTOR_SHIFT, nr_blocks, 1024u * 1024u * 16),
 		  superblock_ref_count_(0)
 	{
@@ -217,14 +231,14 @@ namespace persistent_data {
 
 	template <uint32_t BlockSize>
 	int
-	block_manager<BlockSize>::open_or_create_block_file(string const &path, off_t file_size, mode m)
+	block_manager<BlockSize>::open_or_create_block_file(string const &path, off_t file_size, mode m, bool excl)
 	{
 		switch (m) {
 		case READ_ONLY:
-			return open_block_file(path, file_size, false);
+			return open_block_file(path, file_size, false, excl);
 
 		case READ_WRITE:
-			return open_block_file(path, file_size, true);
+			return open_block_file(path, file_size, true, excl);
 
 		case CREATE:
 			return create_block_file(path, file_size);

persistent-data/data-structures/btree.h

@@ -22,6 +22,7 @@
 #include "base/endian_utils.h"
 #include "persistent-data/transaction_manager.h"
 #include "persistent-data/data-structures/ref_counter.h"
+#include "persistent-data/data-structures/btree_disk_structures.h"
 
 #include <boost/noncopyable.hpp>
 #include <boost/optional.hpp>
@@ -61,36 +62,6 @@ namespace persistent_data {
 		using namespace base;
 		using namespace std;
 
-		uint32_t const BTREE_CSUM_XOR = 121107;
-
-		//------------------------------------------------
-		// On disk data layout for btree nodes
-		enum node_flags {
-			INTERNAL_NODE = 1,
-			LEAF_NODE = 1 << 1
-		};
-
-		struct node_header {
-			le32 csum;
-			le32 flags;
-			le64 blocknr; /* which block this node is supposed to live in */
-
-			le32 nr_entries;
-			le32 max_entries;
-			le32 value_size;
-			le32 padding;
-		} __attribute__((packed));
-
-		struct disk_node {
-			struct node_header header;
-			le64 keys[0];
-		} __attribute__((packed));
-
-		enum node_type {
-			INTERNAL,
-			LEAF
-		};
-
 		//------------------------------------------------
 		// Class that acts as an interface over the raw little endian btree
 		// node data.
@@ -161,6 +132,9 @@ namespace persistent_data {
 				return raw_;
 			}
 
+			bool value_sizes_match() const;
+			std::string value_mismatch_string() const;
+
 		private:
 			static unsigned calc_max_entries(void);
 			void check_fits_within_block() const;

persistent-data/data-structures/btree.tcc

@@ -21,6 +21,7 @@
 #include "persistent-data/errors.h"
 #include "persistent-data/checksum.h"
 #include "persistent-data/transaction_manager.h"
+#include "persistent-data/validators.h"
 
 #include <iostream>
 
@@ -32,29 +33,6 @@ namespace {
 	using namespace btree_detail;
 	using namespace std;
 
-	struct btree_node_validator : public bcache::validator {
-		virtual void check(void const *raw, block_address location) const {
-			disk_node const *data = reinterpret_cast<disk_node const *>(raw);
-			node_header const *n = &data->header;
-			crc32c sum(BTREE_CSUM_XOR);
-			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
-			if (sum.get_sum() != to_cpu<uint32_t>(n->csum))
-				throw checksum_error("bad checksum in btree node");
-
-			if (to_cpu<uint64_t>(n->blocknr) != location)
-				throw checksum_error("bad block nr in btree node");
-		}
-
-		virtual void prepare(void *raw, block_address location) const {
-			disk_node *data = reinterpret_cast<disk_node *>(raw);
-			node_header *n = &data->header;
-			n->blocknr = to_disk<base::le64, uint64_t>(location);
-
-			crc32c sum(BTREE_CSUM_XOR);
-			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
-			n->csum = to_disk<base::le32>(sum.get_sum());
-		}
-	};
 }
 
 //----------------------------------------------------------------
@@ -362,19 +340,31 @@ namespace persistent_data {
 		}
 	}
 
+	template <typename ValueTraits>
+	bool
+	node_ref<ValueTraits>::value_sizes_match() const {
+		return sizeof(typename ValueTraits::disk_type) == get_value_size();
+	}
+
+	template <typename ValueTraits>
+	std::string
+	node_ref<ValueTraits>::value_mismatch_string() const {
+		std::ostringstream out;
+		out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type)
+		    << ", but got " << get_value_size()
+		    << ".  This is not the btree you are looking for." << std::endl;
+
+		return out.str();
+	}
+
 	template <typename ValueTraits>
 	void
 	node_ref<ValueTraits>::check_fits_within_block() const {
 		if (checked_)
 			return;
 
-		if (sizeof(typename ValueTraits::disk_type) != get_value_size()) {
-			std::ostringstream out;
-			out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type)
-			    << ", but got " << get_value_size()
-			    << ".  This is not the btree you are looking for." << std::endl;
-			throw std::runtime_error(out.str());
-		}
+		if (!value_sizes_match())
+			throw std::runtime_error(value_mismatch_string());
 
 		unsigned max = calc_max_entries();
 
@@ -398,7 +388,7 @@ namespace persistent_data {
 		  destroy_(false),
 		  internal_rc_(tm.get_sm()),
 		  rc_(rc),
-		  validator_(new btree_node_validator)
+		  validator_(create_btree_node_validator())
 	{
 		using namespace btree_detail;
 
@@ -432,7 +422,7 @@ namespace persistent_data {
 		  root_(root),
 		  internal_rc_(tm.get_sm()),
 		  rc_(rc),
-		  validator_(new btree_node_validator)
+		  validator_(create_btree_node_validator())
 	{
 	}
 
@@ -446,7 +436,8 @@ namespace persistent_data {
 		template <typename ValueTraits>
 		struct lower_bound_search {
 			static boost::optional<unsigned> search(btree_detail::node_ref<ValueTraits> n, uint64_t key) {
-				return n.lower_bound(key);
+				int i = n.lower_bound(key);
+				return (i < 0) ? boost::optional<unsigned>() : boost::optional<unsigned>(i);
 			}
 		};
 
@@ -595,9 +586,13 @@ namespace persistent_data {
 
 			}
 
-			mi = leaf.lower_bound(key);
-			if (!mi || *mi < 0)
-				return boost::optional<leaf_type>();
+			{
+				int lb = leaf.lower_bound(key);
+				if (lb < 0)
+					return boost::optional<leaf_type>();
+
+				mi = lb;
+			}
 
 			node_ref<block_traits> internal = spine.template get_node<block_traits>();
 			block = internal.value_at(*mi);

persistent-data/data-structures/btree_damage_visitor.h

@@ -70,10 +70,17 @@ namespace persistent_data {
 			}
 
 			maybe_run64 end() {
+				maybe_run64 r;
+
 				if (damaged_)
-					return maybe_run64(damage_begin_);
+					r = maybe_run64(damage_begin_);
 				else
-					return maybe_run64();
+					r = maybe_run64();
+
+				damaged_ = false;
+				damage_begin_ = 0;
+
+				return r;
 			}
 
 		private:
@@ -190,6 +197,7 @@ namespace persistent_data {
 
 			error_outcome error_accessing_node(node_location const &l, block_address b,
 							   std::string const &what) {
+				update_path(l.path);
 				report_damage(what);
 				return btree<Levels, ValueTraits>::visitor::EXCEPTION_HANDLED;
 			}
@@ -210,6 +218,7 @@ namespace persistent_data {
 					    btree_detail::node_ref<block_traits> const &n) {
 				if (!already_visited(n) &&
 				    check_block_nr(n) &&
+				    check_value_size(n) &&
 				    check_max_entries(n) &&
 				    check_nr_entries(n, loc.is_sub_root()) &&
 				    check_ordered_keys(n) &&
@@ -229,6 +238,7 @@ namespace persistent_data {
 					btree_detail::node_ref<ValueTraits2> const &n) {
 				if (!already_visited(n) &&
 				    check_block_nr(n) &&
+				    check_value_size(n) &&
 				    check_max_entries(n) &&
 				    check_nr_entries(n, loc.is_sub_root()) &&
 				    check_ordered_keys(n) &&
@@ -275,6 +285,16 @@ namespace persistent_data {
 				return true;
 			}
 
+			template <typename node>
+			bool check_value_size(node const &n) {
+				if (!n.value_sizes_match()) {
+					report_damage(n.value_mismatch_string());
+					return false;
+				}
+
+				return true;
+			}
+
 			template <typename node>
 			bool check_max_entries(node const &n) {
 				size_t elt_size = sizeof(uint64_t) + n.get_value_size();

persistent-data/data-structures/btree_disk_structures.h

@@ -0,0 +1,64 @@
+// Copyright (C) 2011 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef PERSISTENT_DATA_BTREE_DISK_STRUCTURES_H
+#define PERSISTENT_DATA_BTREE_DISK_STRUCTURES_H
+
+#include "base/endian_utils.h"
+
+//----------------------------------------------------------------
+
+namespace persistent_data {
+	namespace btree_detail {
+		using namespace base;
+
+		uint32_t const BTREE_CSUM_XOR = 121107;
+
+		//------------------------------------------------
+		// On disk data layout for btree nodes
+		enum node_flags {
+			INTERNAL_NODE = 1,
+			LEAF_NODE = 1 << 1
+		};
+
+		struct node_header {
+			le32 csum;
+			le32 flags;
+			le64 blocknr; /* which block this node is supposed to live in */
+
+			le32 nr_entries;
+			le32 max_entries;
+			le32 value_size;
+			le32 padding;
+		} __attribute__((packed));
+
+		struct disk_node {
+			struct node_header header;
+			le64 keys[0];
+		} __attribute__((packed));
+
+		enum node_type {
+			INTERNAL,
+			LEAF
+		};
+	}
+}
+
+//----------------------------------------------------------------
+
+#endif

persistent-data/file_utils.cc

@@ -48,10 +48,10 @@ persistent_data::get_nr_blocks(string const &path)
 }
 
 persistent_data::block_manager<>::ptr
-persistent_data::open_bm(std::string const &dev_path, block_manager<>::mode m)
+persistent_data::open_bm(std::string const &dev_path, block_manager<>::mode m, bool excl)
 {
 	block_address nr_blocks = get_nr_blocks(dev_path);
-	return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m));
+	return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m, excl));
 }
 
 void

persistent-data/file_utils.h

@@ -10,7 +10,8 @@
 // FIXME: move to a different unit
 namespace persistent_data {
 	persistent_data::block_address get_nr_blocks(string const &path);
-	block_manager<>::ptr open_bm(std::string const &dev_path, block_manager<>::mode m);
+	block_manager<>::ptr open_bm(std::string const &dev_path,
+				     block_manager<>::mode m, bool excl = true);
 
 	void check_file_exists(std::string const &file);
 }

persistent-data/space-maps/disk.cc

@@ -747,6 +747,11 @@ persistent_data::create_metadata_sm(transaction_manager &tm, block_address nr_bl
 {
 	index_store::ptr store(new metadata_index_store(tm));
 	checked_space_map::ptr sm(new sm_disk(store, tm));
+
+	if (nr_blocks > MAX_METADATA_BLOCKS) {
+		cerr << "truncating metadata device to " << MAX_METADATA_BLOCKS << " 4k blocks\n";
+		nr_blocks = MAX_METADATA_BLOCKS;
+	}
 	sm->extend(nr_blocks);
 	sm->commit();
 	return create_careful_alloc_sm(

persistent-data/space-maps/disk_structures.h

@@ -61,6 +61,7 @@ namespace persistent_data {
 		};
 
 		unsigned const MAX_METADATA_BITMAPS = 255;
+		unsigned const MAX_METADATA_BLOCKS = (255 * ((1 << 14) - 64));
 		unsigned const ENTRIES_PER_BYTE = 4;
 
 		struct metadata_index {

persistent-data/validators.cc

@@ -0,0 +1,53 @@
+#include "persistent-data/block.h"
+#include "persistent-data/checksum.h"
+#include "persistent-data/data-structures/btree_disk_structures.h"
+#include "persistent-data/errors.h"
+#include "persistent-data/validators.h"
+
+using namespace bcache;
+using namespace persistent_data;
+
+//----------------------------------------------------------------
+
+namespace {
+	using namespace btree_detail;
+
+	struct btree_node_validator : public bcache::validator {
+		virtual void check(void const *raw, block_address location) const {
+			disk_node const *data = reinterpret_cast<disk_node const *>(raw);
+			node_header const *n = &data->header;
+			crc32c sum(BTREE_CSUM_XOR);
+			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
+			if (sum.get_sum() != to_cpu<uint32_t>(n->csum)) {
+				std::ostringstream out;
+				out << "bad checksum in btree node (block " << location << ")";
+				throw checksum_error(out.str());
+			}
+
+			if (to_cpu<uint64_t>(n->blocknr) != location) {
+				std::ostringstream out;
+				out << "bad block nr in btree node (block = " << location << ")";
+				throw checksum_error(out.str());
+			}
+		}
+
+		virtual void prepare(void *raw, block_address location) const {
+			disk_node *data = reinterpret_cast<disk_node *>(raw);
+			node_header *n = &data->header;
+			n->blocknr = to_disk<base::le64, uint64_t>(location);
+
+			crc32c sum(BTREE_CSUM_XOR);
+			sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
+			n->csum = to_disk<base::le32>(sum.get_sum());
+		}
+	};
+}
+
+//----------------------------------------------------------------
+
+bcache::validator::ptr persistent_data::create_btree_node_validator()
+{
+	return bcache::validator::ptr(new btree_node_validator());
+}
+
+//----------------------------------------------------------------

persistent-data/validators.h

@@ -0,0 +1,14 @@
+#ifndef PERSISTENT_DATA_VALIDATORS_H
+#define PERSISTENT_DATA_VALIDATORS_H
+
+#include "block-cache/block_cache.h"
+
+//----------------------------------------------------------------
+
+namespace persistent_data {
+	bcache::validator::ptr create_btree_node_validator();
+}
+
+//----------------------------------------------------------------
+
+#endif