[data-structures] array_block

persistent-data/data-structures/array_block.h

@@ -0,0 +1,188 @@
+// Copyright (C) 2012 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef ARRAY_BLOCK_H
+#define ARRAY_BLOCK_H
+
+#include "persistent-data/endian_utils.h"
+
+//----------------------------------------------------------------
+
+namespace persistent_data {
+	struct array_block_disk {
+		base::__le32 csum;
+		base::__le32 max_entries;
+		base::__le32 nr_entries;
+		base::__le32 value_size;
+		base::__le64 blocknr;
+	} __attribute__((packed));
+
+	// RefType should be either a read_ref or write_ref from block_manager
+	template <typename ValueTraits, typename RefType>
+	class array_block {
+	public:
+		typedef boost::shared_ptr<array_block> ptr;
+		typedef typename ValueTraits::disk_type disk_type;
+		typedef typename ValueTraits::value_type value_type;
+		typedef typename ValueTraits::ref_counter ref_counter;
+
+		array_block(RefType ref,
+			    ref_counter rc,
+			    uint32_t value_size)
+			: ref_(ref),
+			  rc_(rc) {
+
+			using namespace base;
+			struct array_block_disk *header = get_header();
+			header->max_entries = to_disk<__le32>(calc_max_entries(value_size));
+			header->nr_entries = to_disk<__le32>(static_cast<uint32_t>(0));
+			header->value_size = to_disk<__le32>(value_size);
+		}
+
+		array_block(RefType ref, ref_counter rc)
+			: ref_(ref),
+			  rc_(rc) {
+		}
+
+		uint32_t max_entries() const {
+			return base::to_cpu<uint32_t>(get_header()->max_entries);
+		}
+
+		uint32_t nr_entries() const {
+			return base::to_cpu<uint32_t>(get_header()->nr_entries);
+		}
+
+		uint32_t value_size() const {
+			return base::to_cpu<uint32_t>(get_header()->value_size);
+		}
+
+		void grow(uint32_t nr, value_type const &default_value) {
+			uint32_t old_nr = nr_entries();
+
+			if (nr >= max_entries())
+				throw runtime_error("array_block index out of bounds");
+
+			if (nr <= old_nr)
+				throw runtime_error("array_block grow method called with smaller size");
+
+			grow_(nr, default_value);
+		}
+
+		void shrink(uint32_t nr) {
+			uint32_t old_nr = nr_entries();
+
+			if (nr >= old_nr)
+				throw runtime_error("array_block shrink called with larger size");
+
+			shrink_(nr);
+		}
+
+		value_type get(unsigned index) const {
+			value_type v;
+
+			ValueTraits::unpack(element_at(index), v);
+			return v;
+		}
+
+		void set(unsigned index, value_type const &new_value) {
+			value_type const old_value = get(index);
+			rc_.inc(new_value);
+			ValueTraits::pack(new_value, element_at(index));
+			rc_.dec(old_value);
+		}
+
+		void inc_all_entries() {
+			unsigned e = nr_entries();
+
+			for (unsigned index = 0; index < e; index++)
+				rc_.inc(get(index));
+		}
+
+		void dec_all_entries() {
+			unsigned e = nr_entries();
+
+			for (unsigned index = 0; index < e; index++)
+				rc_.dec(get(index));
+		}
+
+		ref_counter const &get_ref_counter() const {
+			return rc_;
+		}
+
+	private:
+		static uint32_t calc_max_entries(uint32_t value_size) {
+			return (RefType::BLOCK_SIZE - sizeof(array_block_disk)) / value_size;
+		}
+
+		void set_nr_entries(uint32_t nr) {
+			using namespace base;
+			array_block_disk *h = get_header();
+			h->nr_entries = to_disk<__le32>(nr);
+		}
+
+		void grow_(uint32_t nr, value_type const &default_value) {
+			uint32_t old_nr_entries = nr_entries();
+			set_nr_entries(nr);
+
+			for (unsigned i = old_nr_entries; i < nr; i++) {
+				ValueTraits::pack(default_value, element_at(i));
+				rc_.inc(default_value);
+			}
+		}
+
+		void shrink_(uint32_t nr) {
+			for (unsigned i = nr_entries() - 1; i >= nr; i--)
+				rc_.dec(get(i));
+
+			set_nr_entries(nr);
+		}
+
+		array_block_disk *get_header() {
+			return reinterpret_cast<array_block_disk *>(ref_.data().raw());
+		}
+
+		array_block_disk const *get_header() const {
+			return reinterpret_cast<array_block_disk const *>(ref_.data().raw());
+		}
+
+		disk_type &element_at(unsigned int index) {
+			if (index >= nr_entries())
+				throw runtime_error("array_block index out of bounds");
+
+			array_block_disk *a = get_header();
+			disk_type *elts = reinterpret_cast<disk_type *>(a + 1);
+			return elts[index];
+		}
+
+		disk_type const &element_at(unsigned int index) const {
+			if (index >= nr_entries())
+				throw runtime_error("array_block index out of bounds");
+
+			array_block_disk const *a = get_header();
+			disk_type const *elts = reinterpret_cast<disk_type const *>(a + 1);
+			return elts[index];
+		}
+
+		RefType ref_;
+		ref_counter rc_;
+	};
+}
+
+//----------------------------------------------------------------
+
+#endif

unit-tests/Makefile.in

@@ -17,7 +17,7 @@
 # <http://www.gnu.org/licenses/>.
 
 TEST_SOURCE=\
-	unit-tests/array_t.cc \
+	unit-tests/array_block_t.cc \
 	unit-tests/buffer_t.cc \
 	unit-tests/cache_t.cc \
 	unit-tests/block_t.cc \
@@ -29,11 +29,11 @@ TEST_SOURCE=\
 TEST_PROGRAMS=$(subst .cc,,$(TEST_SOURCE))
 
 unit-test: $(TEST_PROGRAMS)
-	for p in $(TEST_PROGRAMS); do echo Running $$p; ./$$p; done
+	r=0; for p in $(TEST_PROGRAMS); do echo Running $$p; ./$$p; [ $$? -ne 0 ] && r=1; done; exit $$r
 
 .PHONY: unit-test
 
-unit-tests/array_t: unit-tests/array_t.o $(OBJECTS)
+unit-tests/array_block_t: unit-tests/array_block_t.o $(OBJECTS)
 	g++ $(CXXFLAGS) $(INCLUDES) -o $@ $+ $(LIBS) $(LIBEXPAT)
 
 unit-tests/buffer_t: unit-tests/buffer_t.o $(OBJECTS)

unit-tests/array_block_t.cc

@@ -0,0 +1,246 @@
+// Copyright (C) 2013 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include "persistent-data/transaction_manager.h"
+#include "persistent-data/space-maps/core.h"
+#include "persistent-data/data-structures/array_block.h"
+#include "test_utils.h"
+
+#include <utility>
+#include <vector>
+
+#define BOOST_TEST_MODULE ArrayBlockTests
+#include <boost/test/included/unit_test.hpp>
+
+using namespace boost;
+using namespace persistent_data;
+using namespace std;
+using namespace test;
+
+//----------------------------------------------------------------
+
+namespace {
+	uint64_t MAX_VALUE = 1000ull;
+	block_address const NR_BLOCKS = 1024;
+	typedef typename block_manager<>::noop_validator noop_validator;
+	typedef typename block_manager<>::read_ref read_ref;
+	typedef typename block_manager<>::write_ref write_ref;
+
+	// FIXME: lift to utils?
+	class simple_ref_counter {
+	public:
+		simple_ref_counter(uint64_t nr_counts)
+			: counts_(nr_counts, 0u) {
+		}
+
+		void inc(uint64_t n) {
+			counts_.at(n)++;
+		}
+
+		void dec(uint64_t n) {
+			counts_.at(n)--;
+		}
+
+		unsigned get(uint64_t n) const {
+			return counts_.at(n);
+		}
+
+	private:
+		vector<unsigned> counts_;
+	};
+
+	struct uint64_traits {
+		typedef base::__le64 disk_type;
+		typedef uint64_t value_type;
+		typedef simple_ref_counter ref_counter;
+
+		static void unpack(disk_type const &disk, value_type &value) {
+			value = base::to_cpu<uint64_t>(disk);
+		}
+
+		static void pack(value_type const &value, disk_type &disk) {
+			disk = base::to_disk<base::__le64>(value);
+		}
+	};
+
+	typedef array_block<uint64_traits, write_ref> ablock64;
+	typedef array_block<uint64_traits, read_ref> ablock64_r;
+
+	block_manager<>::validator::ptr
+	validator() {
+		return block_manager<>::validator::ptr(new block_manager<>::noop_validator);
+	}
+
+	transaction_manager::ptr
+	create_tm() {
+		block_manager<>::ptr bm = create_bm<4096>(NR_BLOCKS);
+		space_map::ptr sm(new core_map(NR_BLOCKS));
+		transaction_manager::ptr tm(new transaction_manager(bm, sm));
+		return tm;
+	}
+
+	pair<ablock64, block_address>
+	new_array_block(transaction_manager::ptr tm) {
+		uint64_traits::ref_counter rc(MAX_VALUE);
+		write_ref wr = tm->new_block(validator());
+		return make_pair(ablock64(wr, rc, sizeof(uint64_t)), wr.get_location());
+	}
+
+	ablock64
+	open_array_block(transaction_manager::ptr tm, block_address loc) {
+		uint64_traits::ref_counter rc(MAX_VALUE);
+		pair<write_ref, bool> p = tm->shadow(loc, validator());
+		BOOST_CHECK(!p.second);
+		return ablock64(p.first, rc);
+	}
+
+	ablock64_r
+	read_array_block(transaction_manager::ptr tm, block_address loc) {
+		uint64_traits::ref_counter rc(MAX_VALUE);
+		read_ref rr = tm->read_lock(loc, validator());
+		return ablock64_r(rr, rc);
+	}
+}
+
+//----------------------------------------------------------------
+
+BOOST_AUTO_TEST_CASE(can_create_an_empty_array)
+{
+	block_address loc;
+	transaction_manager::ptr tm = create_tm();
+
+	{
+		pair<ablock64, block_address> p = new_array_block(tm);
+		ablock64 &b = p.first;
+		loc = p.second;
+
+		BOOST_CHECK_EQUAL(b.nr_entries(), 0);
+		BOOST_CHECK_EQUAL(b.value_size(), sizeof(uint64_t));
+		BOOST_CHECK_EQUAL(b.max_entries(), (4096 - 24) / 8);
+
+		BOOST_CHECK_THROW(b.get(0), runtime_error);
+		BOOST_CHECK_THROW(b.set(0, 12345LL), runtime_error);
+	}
+
+	{
+		ablock64 b = open_array_block(tm, loc);
+
+		BOOST_CHECK_EQUAL(b.nr_entries(), 0);
+		BOOST_CHECK_EQUAL(b.value_size(), sizeof(uint64_t));
+		BOOST_CHECK_EQUAL(b.max_entries(), (4096 - 24) / 8);
+
+		BOOST_CHECK_THROW(b.get(0), runtime_error);
+		BOOST_CHECK_THROW(b.set(0, 12345LL), runtime_error);
+	}
+}
+
+BOOST_AUTO_TEST_CASE(read_only_array_blocks_are_possible)
+{
+	block_address loc;
+	transaction_manager::ptr tm = create_tm();
+
+	{
+		pair<ablock64, block_address> p = new_array_block(tm);
+		ablock64 &b = p.first;
+		loc = p.second;
+
+		BOOST_CHECK_EQUAL(b.nr_entries(), 0);
+		BOOST_CHECK_EQUAL(b.value_size(), sizeof(uint64_t));
+		BOOST_CHECK_EQUAL(b.max_entries(), (4096 - 24) / 8);
+
+		BOOST_CHECK_THROW(b.get(0), runtime_error);
+		BOOST_CHECK_THROW(b.set(0, 12345LL), runtime_error);
+	}
+
+	{
+		ablock64_r b = read_array_block(tm, loc);
+
+		BOOST_CHECK_EQUAL(b.nr_entries(), 0);
+		BOOST_CHECK_EQUAL(b.value_size(), sizeof(uint64_t));
+		BOOST_CHECK_EQUAL(b.max_entries(), (4096 - 24) / 8);
+
+		BOOST_CHECK_THROW(b.get(0), runtime_error);
+
+		// Compile time error as expected
+		// BOOST_CHECK_THROW(b.set(0, 12345LL), runtime_error);
+	}
+}
+
+BOOST_AUTO_TEST_CASE(growing)
+{
+	uint64_t default_value = 123, new_value = 234;
+	transaction_manager::ptr tm = create_tm();
+	pair<ablock64, block_address> p = new_array_block(tm);
+	ablock64 &b = p.first;
+
+	for (unsigned i = 1; i < b.max_entries(); i++) {
+		BOOST_CHECK_THROW(b.get(i - 1), runtime_error);
+
+		b.grow(i, default_value);
+		BOOST_CHECK_EQUAL(b.get(i - 1), default_value);
+
+		b.set(i - 1, new_value);
+		BOOST_CHECK_EQUAL(b.get(i - 1), new_value);
+
+		BOOST_CHECK_THROW(b.grow(i - 1, default_value), runtime_error);
+	}
+}
+
+BOOST_AUTO_TEST_CASE(shrinking)
+{
+	uint64_t default_value = 123;
+	transaction_manager::ptr tm = create_tm();
+	pair<ablock64, block_address> p = new_array_block(tm);
+	ablock64 &b = p.first;
+
+	b.grow(b.max_entries() - 1, default_value);
+	for (unsigned i = b.max_entries() - 2; i; i--) {
+		BOOST_CHECK_EQUAL(b.get(i - 1), default_value);
+		b.shrink(i);
+		BOOST_CHECK_THROW(b.get(i), runtime_error);
+		BOOST_CHECK_THROW(b.shrink(i), runtime_error);
+	}
+}
+
+BOOST_AUTO_TEST_CASE(ref_counting)
+{
+	transaction_manager::ptr tm = create_tm();
+	pair<ablock64, block_address> p = new_array_block(tm);
+	ablock64 &b = p.first;
+	simple_ref_counter const &rc = b.get_ref_counter();
+
+	BOOST_CHECK_EQUAL(rc.get(123), 0);
+	b.grow(b.max_entries() - 1, 123);
+	BOOST_CHECK_EQUAL(rc.get(123), b.max_entries() - 1);
+
+	b.shrink(100);
+	BOOST_CHECK_EQUAL(rc.get(123), 100);
+
+	b.set(1, 0);
+	b.set(2, 2);
+	BOOST_CHECK_EQUAL(rc.get(123), 98);
+	BOOST_CHECK_EQUAL(rc.get(0), 1);
+
+	b.set(2, 2);
+	BOOST_CHECK_EQUAL(rc.get(2), 1);
+	b.set(10, 2);
+	BOOST_CHECK_EQUAL(rc.get(2), 2);
+	BOOST_CHECK_EQUAL(rc.get(123), 97);
+}
+
+//----------------------------------------------------------------

unit-tests/block_t.cc

@@ -17,31 +17,18 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "persistent-data/block.h"
+#include "test_utils.h"
 
 #define BOOST_TEST_MODULE BlockManagerTests
 #include <boost/test/included/unit_test.hpp>
 #include <stdlib.h>
 
 using namespace std;
+using namespace test;
 
 //----------------------------------------------------------------
 
 namespace {
-	unsigned const MAX_HELD_LOCKS = 16;
-
-	template <uint32_t BlockSize>
-	typename block_manager<BlockSize>::ptr
-	create_bm(block_address nr = 1024) {
-		string const path("./test.data");
-		int r = system("rm -f ./test.data");
-		if (r < 0)
-			throw runtime_error("couldn't rm -f ./test.data");
-
-		return typename block_manager<BlockSize>::ptr(
-			new block_manager<BlockSize>(path, nr, MAX_HELD_LOCKS,
-						     block_io<BlockSize>::CREATE));
-	}
-
 	template <uint32_t BlockSize>
 	void check_all_bytes(typename block_manager<BlockSize>::read_ref const &rr, int v) {
 		persistent_data::buffer<BlockSize> const &data = rr.data();

unit-tests/test_utils.h

@@ -0,0 +1,40 @@
+// Copyright (C) 2013 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include "persistent-data/block.h"
+
+//----------------------------------------------------------------
+
+namespace test {
+	unsigned const MAX_HELD_LOCKS = 16;
+
+	template <uint32_t BlockSize>
+	typename block_manager<BlockSize>::ptr
+	create_bm(block_address nr = 1024) {
+		string const path("./test.data");
+		int r = system("rm -f ./test.data");
+		if (r < 0)
+			throw runtime_error("couldn't rm -f ./test.data");
+
+		return typename block_manager<BlockSize>::ptr(
+			new block_manager<BlockSize>(path, nr, MAX_HELD_LOCKS,
+						     block_io<BlockSize>::CREATE));
+	}
+}
+
+//----------------------------------------------------------------