Merge pull request #147 from mingnus/thin-generate-metadata-wip

Add unit tests for sequence_generator, and fix bugs

Makefile.in

@@ -44,6 +44,7 @@ SOURCE=\
 	base/file_utils.cc \
 	base/progress_monitor.cc \
 	base/rolling_hash.cc \
+	base/sequence_generator.cc \
 	base/xml_utils.cc \
 	block-cache/block_cache.cc \
 	block-cache/copier.cc \

base/io_generator.cc

@@ -1,5 +1,5 @@
 #include "base/io_generator.h"
-#include "persistent-data/run_set.h"
+#include "base/sequence_generator.h"
 #include <stdexcept>
 #include <cstdlib>
 #include <cstring>
@@ -26,150 +26,6 @@ namespace {
 
 	//--------------------------------
 
-	class sequence_generator {
-	public:
-		typedef std::shared_ptr<sequence_generator> ptr;
-
-		virtual uint64_t next() = 0;
-	};
-
-	// - The maximum generated value is not greater than (begin+size)
-	// - The generated values are not aligned to the step, if the begin
-	//   value is not aligned.
-	class forward_sequence_generator: public sequence_generator {
-	public:
-		forward_sequence_generator(uint64_t begin,
-					   uint64_t size,
-					   uint64_t step)
-			: begin_(begin),
-			  end_(begin + (size / step) * step),
-			  step_(step),
-			  current_(begin),
-			  rounds_(0) {
-			if (size < step)
-				throw std::runtime_error("size must be greater than the step");
-		}
-
-		forward_sequence_generator()
-			: begin_(0), end_(1), step_(1),
-			  current_(0), rounds_(0) {
-		}
-
-		uint64_t next() {
-			uint64_t r = current_;
-			current_ += step_;
-			if (current_ >= end_) {
-				current_ = begin_;
-				++rounds_;
-			}
-			return r;
-		}
-
-		void reset(uint64_t begin, uint64_t size, uint64_t step = 1) {
-			begin_ = begin;
-			end_ = begin + (size / step) * step;
-			step_ = step;
-			current_ = begin;
-			rounds_ = 0;
-		}
-
-		uint64_t get_rounds() {
-			return rounds_;
-		}
-
-	private:
-		uint64_t begin_;
-		uint64_t end_;
-		uint64_t step_;
-		uint64_t current_;
-		uint64_t rounds_;
-	};
-
-	// - The maximum generated value is not greater than (begin+size)
-	// - The generated values are not aligned to the step, if the begin
-	//   value is not aligned.
-	class random_sequence_generator: public sequence_generator {
-	public:
-		// TODO: load random seeds
-		random_sequence_generator(uint64_t begin,
-					  uint64_t size,
-					  uint64_t step,
-					  unsigned seq_nr = 1)
-			: begin_(begin),
-			  nr_steps_(size / step),
-			  step_(step),
-			  max_forward_steps_(seq_nr),
-			  nr_generated_(0)
-		{
-			if (!max_forward_steps_ || max_forward_steps_ > nr_steps_)
-				throw std::runtime_error("invalid number of forward steps");
-
-			if (max_forward_steps_ > 1)
-				reset_forward_generator();
-		}
-
-		uint64_t next() {
-			// FIXME: eliminate if-else
-			uint64_t step_idx = (max_forward_steps_ > 1) ?
-				next_forward_step() : next_random_step();
-			rand_map_.add(step_idx);
-			++nr_generated_;
-
-			// wrap-around
-			if (nr_generated_ == nr_steps_) {
-				rand_map_.clear();
-				nr_generated_ = 0;
-			}
-
-			return begin_ + step_idx * step_;
-		}
-
-	private:
-		void reset_forward_generator() {
-			uint64_t begin = next_random_step();
-			unsigned seq_nr = (std::rand() % max_forward_steps_) + 1;
-			forward_gen_.reset(begin, seq_nr);
-		}
-
-		uint64_t next_forward_step() {
-			uint64_t step_idx;
-
-			bool found = true;
-			while (found) {
-				step_idx = forward_gen_.next();
-				found = rand_map_.member(step_idx);
-
-				if (found || forward_gen_.get_rounds())
-					reset_forward_generator();
-			}
-
-			return step_idx;
-		}
-
-		uint64_t next_random_step() const {
-			uint64_t step_idx;
-
-			bool found = true;
-			while (found) {
-				step_idx = std::rand() % nr_steps_;
-				found = rand_map_.member(step_idx);
-			}
-
-			return step_idx;
-		}
-
-		uint64_t begin_;
-		uint64_t nr_steps_;
-		uint64_t step_;
-		unsigned max_forward_steps_;
-
-		base::run_set<uint64_t> rand_map_;
-		uint64_t nr_generated_;
-		forward_sequence_generator forward_gen_;
-	};
-
-	//--------------------------------
-
 	class op_generator {
 	public:
 		typedef std::shared_ptr<op_generator> ptr;
@@ -242,17 +98,17 @@ namespace {
 
 	sequence_generator::ptr
 	base_io_generator::create_offset_generator(io_generator_options const &opts) {
-		sequence_generator *gen;
+		sequence_generator::ptr gen;
 
 		if (opts.pattern_.is_random())
-			gen = new random_sequence_generator(opts.offset_,
+			gen = create_random_sequence_generator(opts.offset_,
 					opts.size_, opts.block_size_,
 					opts.nr_seq_blocks_);
 		else
-			gen = new forward_sequence_generator(opts.offset_,
+			gen = create_forward_sequence_generator(opts.offset_,
 					opts.size_, opts.block_size_);
 
-		return sequence_generator::ptr(gen);
+		return gen;
 	}
 
 	op_generator::ptr

base/run.h

@@ -1,5 +1,5 @@
-#ifndef PERSISTENT_DATA_RANGE_H
+#ifndef BASE_DATA_RANGE_H
-#define PERSISTENT_DATA_RANGE_H
+#define BASE_DATA_RANGE_H
 
 #include <boost/optional.hpp>
 #include <ostream>

base/run_set.h

@@ -1,7 +1,7 @@
-#ifndef PERSISTENT_DATA_H
+#ifndef BASE_RUN_SET_H
-#define PERSISTENT_DATA_H
+#define BASE_RUN_SET_H
 
-#include "persistent-data/run.h"
+#include "base/run.h"
 
 #include <algorithm>
 #include <set>
@@ -92,6 +92,14 @@ namespace base {
 			return runs_.end();
 		}
 
+		const_iterator lower_bound(T const &b) const {
+			return runs_.lower_bound(run<T>(b, b + 1));
+		}
+
+		const_iterator upper_bound(T const &b) const {
+			return runs_.upper_bound(run<T>(b, b + 1));
+		}
+
 		void negate() {
 			rset replacement;
 

base/sequence_generator.cc

@@ -0,0 +1,195 @@
+#include "base/run_set.h"
+#include "base/sequence_generator.h"
+#include <stdexcept>
+
+//----------------------------------------------------------------
+
+namespace {
+	// - The maximum generated value is not greater than (begin+size)
+	// - The generated values are not aligned to the step, if the begin
+	//   value is not aligned.
+	class forward_sequence_generator: public base::sequence_generator {
+	public:
+		forward_sequence_generator(uint64_t begin,
+					   uint64_t size,
+					   uint64_t step)
+			: begin_(begin),
+			  step_(step),
+			  current_(begin),
+			  rounds_(0)
+		{
+			verify_parameters(size, step);
+			end_ = begin + (size / step) * step;
+		}
+
+		forward_sequence_generator()
+			: begin_(0), end_(1), step_(1),
+			  current_(0), rounds_(0) {
+		}
+
+		uint64_t next() {
+			uint64_t r = current_;
+			current_ += step_;
+			if (current_ >= end_) {
+				current_ = begin_;
+				++rounds_;
+			}
+			return r;
+		}
+
+		void reset(uint64_t begin, uint64_t size, uint64_t step = 1) {
+			verify_parameters(size, step);
+
+			begin_ = begin;
+			end_ = begin + (size / step) * step;
+			step_ = step;
+			current_ = begin;
+			rounds_ = 0;
+		}
+
+		static void verify_parameters(uint64_t size, uint64_t step) {
+			if (!size || !step)
+				throw std::runtime_error("size or step must be non-zero");
+
+			if (size < step)
+				throw std::runtime_error("size must be greater than the step");
+		}
+
+		uint64_t get_rounds() {
+			return rounds_;
+		}
+
+	private:
+		uint64_t begin_;
+		uint64_t end_;
+		uint64_t step_;
+		uint64_t current_;
+		uint64_t rounds_;
+	};
+
+	// - The maximum generated value is not greater than (begin+size)
+	// - The generated values are not aligned to the step, if the begin
+	//   value is not aligned.
+	class random_sequence_generator: public base::sequence_generator {
+	public:
+		// TODO: load random seeds
+		random_sequence_generator(uint64_t begin,
+					  uint64_t size,
+					  uint64_t step,
+					  unsigned seq_nr = 1)
+			: begin_(begin),
+			  step_(step),
+			  max_forward_steps_(seq_nr),
+			  nr_generated_(0)
+		{
+			if (!size || !step || !seq_nr)
+				throw std::runtime_error("size, step, or forward steps must be non-zero");
+
+			if (size < step)
+				throw std::runtime_error("size must be greater than the step");
+
+			nr_steps_ = size / step;
+
+			if (!max_forward_steps_ || max_forward_steps_ > nr_steps_)
+				throw std::runtime_error("invalid number of forward steps");
+
+			if (max_forward_steps_ > 1)
+				reset_forward_generator();
+		}
+
+		uint64_t next() {
+			// FIXME: eliminate if-else
+			uint64_t step_idx = (max_forward_steps_ > 1) ?
+				next_forward_step() : next_random_step();
+
+			return begin_ + step_idx * step_;
+		}
+
+	private:
+		void reset_forward_generator() {
+			uint64_t begin = peek_random_step();
+
+			unsigned seq_nr = (std::rand() % max_forward_steps_) + 1;
+			base::run_set<uint64_t>::const_iterator it = rand_map_.upper_bound(begin);
+			if (it != rand_map_.end())
+				seq_nr = std::min<uint64_t>(seq_nr, *it->begin_ - begin);
+			else
+				seq_nr = std::min<uint64_t>(seq_nr, nr_steps_ - begin);
+
+			forward_gen_.reset(begin, seq_nr);
+		}
+
+		uint64_t next_forward_step() {
+			uint64_t step_idx = forward_gen_.next();
+			consume_random_map(step_idx);
+
+			if (forward_gen_.get_rounds())
+				reset_forward_generator();
+
+			return step_idx;
+		}
+
+		uint64_t next_random_step() {
+			uint64_t step_idx = peek_random_step();
+			consume_random_map(step_idx);
+
+			return step_idx;
+		}
+
+		void consume_random_map(uint64_t step_idx) {
+			rand_map_.add(step_idx);
+
+			++nr_generated_;
+
+			// wrap-around
+			if (nr_generated_ == nr_steps_) {
+				rand_map_.clear();
+				nr_generated_ = 0;
+			}
+		}
+
+		uint64_t peek_random_step() const {
+			uint64_t step_idx;
+
+			bool found = true;
+			while (found) {
+				step_idx = std::rand() % nr_steps_;
+				found = rand_map_.member(step_idx);
+			}
+
+			return step_idx;
+		}
+
+		uint64_t begin_;
+		uint64_t nr_steps_;
+		uint64_t step_;
+		unsigned max_forward_steps_;
+
+		base::run_set<uint64_t> rand_map_;
+		uint64_t nr_generated_;
+		forward_sequence_generator forward_gen_;
+	};
+}
+
+//----------------------------------------------------------------
+
+base::sequence_generator::ptr
+base::create_forward_sequence_generator(uint64_t begin,
+					uint64_t size,
+					uint64_t step)
+{
+	return sequence_generator::ptr(
+			new forward_sequence_generator(begin, size, step));
+}
+
+base::sequence_generator::ptr
+base::create_random_sequence_generator(uint64_t begin,
+				       uint64_t size,
+				       uint64_t step,
+				       unsigned seq_nr)
+{
+	return sequence_generator::ptr(
+			new random_sequence_generator(begin, size, step, seq_nr));
+}
+
+//----------------------------------------------------------------

base/sequence_generator.h

@@ -0,0 +1,27 @@
+#ifndef BASE_SEQUENCE_GENERATOR_H
+#define BASE_SEQUENCE_GENERATOR_H
+
+#include <memory>
+
+//----------------------------------------------------------------
+
+namespace base {
+	class sequence_generator {
+	public:
+		typedef std::shared_ptr<sequence_generator> ptr;
+
+		virtual uint64_t next() = 0;
+	};
+
+	sequence_generator::ptr
+	create_forward_sequence_generator(uint64_t begin, uint64_t size,
+			uint64_t step);
+
+	sequence_generator::ptr
+	create_random_sequence_generator(uint64_t begin, uint64_t size,
+			uint64_t step, unsigned seq_nr = 1);
+}
+
+//----------------------------------------------------------------
+
+#endif

persistent-data/block_counter.h

@@ -19,8 +19,8 @@
 #ifndef BLOCK_COUNTER_H
 #define BLOCK_COUNTER_H
 
+#include "base/run_set.h"
 #include "block.h"
-#include "run_set.h"
 
 //----------------------------------------------------------------
 

persistent-data/data-structures/bitset.h

@@ -19,7 +19,7 @@
 #ifndef PERSISTENT_DATA_DATA_STRUCTURES_BITSET_H
 #define PERSISTENT_DATA_DATA_STRUCTURES_BITSET_H
 
-#include "persistent-data/run.h"
+#include "base/run.h"
 
 //----------------------------------------------------------------
 

persistent-data/data-structures/btree_damage_visitor.h

@@ -1,9 +1,9 @@
 #ifndef PERSISTENT_DATA_DATA_STRUCTURES_DAMAGE_VISITOR_H
 #define PERSISTENT_DATA_DATA_STRUCTURES_DAMAGE_VISITOR_H
 
+#include "base/run.h"
 #include "persistent-data/data-structures/btree.h"
 #include "persistent-data/data-structures/btree_node_checker.h"
-#include "persistent-data/run.h"
 
 //----------------------------------------------------------------
 

persistent-data/space-maps/subtracting_span_iterator.h

@@ -1,8 +1,8 @@
 #ifndef SPAN_ITERATOR_H
 #define SPAN_ITERATOR_H
 
+#include "base/run_set.h"
 #include "persistent-data/space_map.h"
-#include "persistent-data/run_set.h"
 
 #include <set>
 

persistent-data/space_map.h

@@ -19,9 +19,9 @@
 #ifndef SPACE_MAP_H
 #define SPACE_MAP_H
 
+#include "base/run.h"
 #include "persistent-data/block.h"
 #include "persistent-data/block_counter.h"
-#include "persistent-data/run.h"
 
 #include <boost/optional.hpp>
 #include <functional>

thin-provisioning/device_tree.h

@@ -1,8 +1,8 @@
 #ifndef THIN_DEVICE_CHECKER_H
 #define THIN_DEVICE_CHECKER_H
 
+#include "base/run.h"
 #include "persistent-data/data-structures/btree.h"
-#include "persistent-data/run.h"
 
 //----------------------------------------------------------------
 

thin-provisioning/mapping_tree.h

@@ -1,8 +1,8 @@
 #ifndef MAPPING_TREE_H
 #define MAPPING_TREE_H
 
+#include "base/run.h"
 #include "persistent-data/data-structures/btree.h"
-#include "persistent-data/run.h"
 
 //----------------------------------------------------------------
 

thin-provisioning/rmap_visitor.h

@@ -1,7 +1,7 @@
 #ifndef THIN_RMAP_VISITOR_H
 #define THIN_RMAP_VISITOR_H
 
-#include "persistent-data/run.h"
+#include "base/run.h"
 #include "thin-provisioning/mapping_tree.h"
 
 //----------------------------------------------------------------

thin-provisioning/thin_delta.cc

@@ -7,8 +7,8 @@
 #include "version.h"
 
 #include "base/indented_stream.h"
+#include "base/run.h"
 #include "persistent-data/data-structures/btree_damage_visitor.h"
-#include "persistent-data/run.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/space-maps/disk.h"
 #include "persistent-data/file_utils.h"

thin-provisioning/thin_rmap.cc

@@ -6,8 +6,8 @@
 
 #include "version.h"
 
+#include "base/run.h"
 #include "persistent-data/data-structures/btree_damage_visitor.h"
-#include "persistent-data/run.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/file_utils.h"
 #include "thin-provisioning/commands.h"

unit-tests/Makefile.in

@@ -65,6 +65,7 @@ TEST_SOURCE=\
 	unit-tests/rmap_visitor_t.cc \
 	unit-tests/rolling_hash_t.cc \
 	unit-tests/run_set_t.cc \
+	unit-tests/sequence_generator_t.cc \
 	unit-tests/space_map_t.cc \
 	unit-tests/span_iterator_t.cc \
 	unit-tests/transaction_manager_t.cc

unit-tests/btree_damage_visitor_t.cc

@@ -3,10 +3,10 @@
 #include "test_utils.h"
 
 #include "base/endian_utils.h"
+#include "base/run.h"
 #include "persistent-data/data-structures/btree_damage_visitor.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/transaction_manager.h"
-#include "persistent-data/run.h"
 
 using namespace base;
 using namespace std;

unit-tests/run_set_t.cc

@@ -1,6 +1,6 @@
 #include "gmock/gmock.h"
 
-#include "persistent-data/run_set.h"
+#include "base/run_set.h"
 
 #include <memory>
 

unit-tests/sequence_generator_t.cc

@@ -0,0 +1,208 @@
+#include "gmock/gmock.h"
+
+#include "base/sequence_generator.h"
+
+using namespace base;
+using namespace std;
+using namespace testing;
+
+//----------------------------------------------------------------
+
+namespace {
+	class SequenceGeneratorTests : public Test {
+	};
+
+	class ForwardSequenceGeneratorTests : public Test {
+	public:
+		ForwardSequenceGeneratorTests() {
+		}
+
+		void reset(uint64_t begin, uint64_t size, uint64_t step) {
+			gen_ = create_forward_sequence_generator(
+					begin, size, step);
+
+			// the maximum value is expected to be less than
+			// (begin + round_down(size, step))
+			begin_ = begin;
+			end_ = begin + (size / step) * step;
+			step_ = step;
+		}
+
+		void test_full_sequence() {
+			for (uint64_t i = begin_; i < end_; i += step_)
+				ASSERT_EQ(i, gen_->next());
+		}
+
+	private:
+		sequence_generator::ptr gen_;
+		uint64_t begin_;
+		uint64_t end_;
+		uint64_t step_;
+	};
+
+	class RandomSequenceGeneratorTests : public Test {
+		struct span_tracker {
+			span_tracker(uint64_t begin, uint64_t step)
+				: last_(begin),
+				  step_(step),
+				  len_(1),
+				  max_len_(1) {
+			}
+
+			void append(uint64_t n) {
+				if (n == last_ + step_)
+					++len_;
+				else
+					len_ = 1;
+
+				if (len_ > max_len_)
+					max_len_ = len_;
+
+				last_ = n;
+			}
+
+			uint64_t last_;
+			uint64_t step_;
+			unsigned len_;
+			unsigned max_len_;
+		};
+
+	public:
+		RandomSequenceGeneratorTests() {
+		}
+
+		void reset(uint64_t begin, uint64_t size, uint64_t step,
+			   unsigned seq_nr = 1) {
+			gen_ = create_random_sequence_generator(
+					begin, size, step, seq_nr);
+
+			// the maximum value is expected to be less than
+			// (begin + round_down(size, step))
+			begin_ = begin;
+			nr_steps_ = size / step;
+			step_ = step;
+			seq_nr_ = seq_nr;
+		}
+
+		void test_full_sequence() {
+			std::set<uint64_t> values;
+
+			// generate the whole sequence
+			uint64_t n = gen_->next();
+			values.insert(n);
+			span_tracker tracker(n, step_);
+
+			for (uint64_t i = 1; i < nr_steps_; i++) {
+				n = gen_->next();
+				values.insert(n);
+				tracker.append(n);
+			}
+
+			// verify the generated sequence
+			ASSERT_EQ(values.size(), nr_steps_); // assert no duplicates
+
+			set<uint64_t>::const_iterator it = values.begin();
+			uint64_t end = begin_ + nr_steps_ * step_;
+			for (uint64_t v = begin_; v < end; v += step_) {
+				ASSERT_EQ(v, *it);
+				++it;
+			}
+
+			// FIXME: refine this assertion
+			// The maximum span length depends on the randomness.
+			// It should be greater than half of the desired length,
+			// if the probabilities are evenly distributed.
+			if (seq_nr_ > 1) {
+				ASSERT_TRUE(tracker.max_len_ >= seq_nr_ / 2);
+			}
+		}
+
+	private:
+		sequence_generator::ptr gen_;
+		uint64_t begin_;
+		uint64_t nr_steps_;
+		uint64_t step_;
+		unsigned seq_nr_;
+	};
+}
+
+//----------------------------------------------------------------
+
+TEST_F(SequenceGeneratorTests, create_with_valid_params)
+{
+	// step < size
+	create_forward_sequence_generator(0, 10, 2);
+
+	// boundary test: step == size
+	create_forward_sequence_generator(0, 10, 10);
+
+	// step < size
+	create_random_sequence_generator(0, 10, 2);
+
+	// boundary test: step == size
+	create_random_sequence_generator(0, 10, 10);
+
+	// seq_nr < nr_steps
+	create_random_sequence_generator(0, 10, 2, 2);
+
+	// boundary test: seq_nr == nr_steps
+	create_random_sequence_generator(0, 10, 2, 5);
+}
+
+TEST_F(SequenceGeneratorTests, create_with_invalid_params)
+{
+	// zero size or step
+	ASSERT_THROW(create_forward_sequence_generator(0, 1, 0), std::runtime_error);
+	ASSERT_THROW(create_forward_sequence_generator(0, 0, 1), std::runtime_error);
+
+	// step > size
+	ASSERT_THROW(create_forward_sequence_generator(0, 10, 11), std::runtime_error);
+
+	// zero size, step, or seq_nr
+	ASSERT_THROW(create_random_sequence_generator(0, 1, 0), std::runtime_error);
+	ASSERT_THROW(create_random_sequence_generator(0, 0, 1), std::runtime_error);
+	ASSERT_THROW(create_random_sequence_generator(0, 1, 1, 0), std::runtime_error);
+
+	// step > size
+	ASSERT_THROW(create_random_sequence_generator(0, 10, 11), std::runtime_error);
+
+	// seq_nr > nr_steps
+	ASSERT_THROW(create_random_sequence_generator(0, 10, 2, 6), std::runtime_error);
+}
+
+TEST_F(ForwardSequenceGeneratorTests, forward_sequence_aligned)
+{
+	reset(50, 100, 2); // begin & size are aligned to step
+	test_full_sequence();
+	test_full_sequence(); // wrap-around
+}
+
+TEST_F(ForwardSequenceGeneratorTests, forward_sequence_unaligned)
+{
+	reset(50, 100, 7); // begin & size are not aligned to step
+	test_full_sequence();
+	test_full_sequence(); // wrap-around
+}
+
+TEST_F(RandomSequenceGeneratorTests, random_sequence_aligned)
+{
+	reset(50, 100, 2); // begin & size are aligned to step
+	test_full_sequence();
+	test_full_sequence(); // wrap-around
+}
+
+TEST_F(RandomSequenceGeneratorTests, random_sequence_unaligned)
+{
+	reset(50, 100, 7); // begin & size are not aligned to step
+	test_full_sequence();
+	test_full_sequence(); // wrap-around
+}
+
+TEST_F(RandomSequenceGeneratorTests, random_sequence_with_span)
+{
+	reset(50, 10000, 2, 10); // begin & size are aligned to step
+	test_full_sequence();
+	test_full_sequence(); // wrap-around
+}
+
+//----------------------------------------------------------------

unit-tests/span_iterator_t.cc

@@ -1,7 +1,7 @@
 #include "gmock/gmock.h"
 
+#include "base/run_set.h"
 #include "persistent-data/space-maps/subtracting_span_iterator.h"
-#include "persistent-data/run_set.h"
 
 using namespace std;
 using namespace persistent_data;