diff --git a/Makefile.in b/Makefile.in index b3791cd..b1fd4aa 100644 --- a/Makefile.in +++ b/Makefile.in @@ -44,6 +44,7 @@ SOURCE=\ base/file_utils.cc \ base/progress_monitor.cc \ base/rolling_hash.cc \ + base/sequence_generator.cc \ base/xml_utils.cc \ block-cache/block_cache.cc \ block-cache/copier.cc \ diff --git a/base/io_generator.cc b/base/io_generator.cc index 88c2927..918a9ed 100644 --- a/base/io_generator.cc +++ b/base/io_generator.cc @@ -1,5 +1,5 @@ #include "base/io_generator.h" -#include "persistent-data/run_set.h" +#include "base/sequence_generator.h" #include #include #include @@ -26,150 +26,6 @@ namespace { //-------------------------------- - class sequence_generator { - public: - typedef std::shared_ptr ptr; - - virtual uint64_t next() = 0; - }; - - // - The maximum generated value is not greater than (begin+size) - // - The generated values are not aligned to the step, if the begin - // value is not aligned. - class forward_sequence_generator: public sequence_generator { - public: - forward_sequence_generator(uint64_t begin, - uint64_t size, - uint64_t step) - : begin_(begin), - end_(begin + (size / step) * step), - step_(step), - current_(begin), - rounds_(0) { - if (size < step) - throw std::runtime_error("size must be greater than the step"); - } - - forward_sequence_generator() - : begin_(0), end_(1), step_(1), - current_(0), rounds_(0) { - } - - uint64_t next() { - uint64_t r = current_; - current_ += step_; - if (current_ >= end_) { - current_ = begin_; - ++rounds_; - } - return r; - } - - void reset(uint64_t begin, uint64_t size, uint64_t step = 1) { - begin_ = begin; - end_ = begin + (size / step) * step; - step_ = step; - current_ = begin; - rounds_ = 0; - } - - uint64_t get_rounds() { - return rounds_; - } - - private: - uint64_t begin_; - uint64_t end_; - uint64_t step_; - uint64_t current_; - uint64_t rounds_; - }; - - // - The maximum generated value is not greater than (begin+size) - // - The generated values are not aligned to the step, if the begin - // value is not aligned. - class random_sequence_generator: public sequence_generator { - public: - // TODO: load random seeds - random_sequence_generator(uint64_t begin, - uint64_t size, - uint64_t step, - unsigned seq_nr = 1) - : begin_(begin), - nr_steps_(size / step), - step_(step), - max_forward_steps_(seq_nr), - nr_generated_(0) - { - if (!max_forward_steps_ || max_forward_steps_ > nr_steps_) - throw std::runtime_error("invalid number of forward steps"); - - if (max_forward_steps_ > 1) - reset_forward_generator(); - } - - uint64_t next() { - // FIXME: eliminate if-else - uint64_t step_idx = (max_forward_steps_ > 1) ? - next_forward_step() : next_random_step(); - rand_map_.add(step_idx); - ++nr_generated_; - - // wrap-around - if (nr_generated_ == nr_steps_) { - rand_map_.clear(); - nr_generated_ = 0; - } - - return begin_ + step_idx * step_; - } - - private: - void reset_forward_generator() { - uint64_t begin = next_random_step(); - unsigned seq_nr = (std::rand() % max_forward_steps_) + 1; - forward_gen_.reset(begin, seq_nr); - } - - uint64_t next_forward_step() { - uint64_t step_idx; - - bool found = true; - while (found) { - step_idx = forward_gen_.next(); - found = rand_map_.member(step_idx); - - if (found || forward_gen_.get_rounds()) - reset_forward_generator(); - } - - return step_idx; - } - - uint64_t next_random_step() const { - uint64_t step_idx; - - bool found = true; - while (found) { - step_idx = std::rand() % nr_steps_; - found = rand_map_.member(step_idx); - } - - return step_idx; - } - - uint64_t begin_; - uint64_t nr_steps_; - uint64_t step_; - unsigned max_forward_steps_; - - base::run_set rand_map_; - uint64_t nr_generated_; - forward_sequence_generator forward_gen_; - }; - - //-------------------------------- - class op_generator { public: typedef std::shared_ptr ptr; @@ -242,17 +98,17 @@ namespace { sequence_generator::ptr base_io_generator::create_offset_generator(io_generator_options const &opts) { - sequence_generator *gen; + sequence_generator::ptr gen; if (opts.pattern_.is_random()) - gen = new random_sequence_generator(opts.offset_, + gen = create_random_sequence_generator(opts.offset_, opts.size_, opts.block_size_, opts.nr_seq_blocks_); else - gen = new forward_sequence_generator(opts.offset_, + gen = create_forward_sequence_generator(opts.offset_, opts.size_, opts.block_size_); - return sequence_generator::ptr(gen); + return gen; } op_generator::ptr diff --git a/persistent-data/run.h b/base/run.h similarity index 94% rename from persistent-data/run.h rename to base/run.h index 78db2a5..b3dcc55 100644 --- a/persistent-data/run.h +++ b/base/run.h @@ -1,5 +1,5 @@ -#ifndef PERSISTENT_DATA_RANGE_H -#define PERSISTENT_DATA_RANGE_H +#ifndef BASE_DATA_RANGE_H +#define BASE_DATA_RANGE_H #include #include diff --git a/persistent-data/run_set.h b/base/run_set.h similarity index 91% rename from persistent-data/run_set.h rename to base/run_set.h index 6c23dc1..3a1a769 100644 --- a/persistent-data/run_set.h +++ b/base/run_set.h @@ -1,7 +1,7 @@ -#ifndef PERSISTENT_DATA_H -#define PERSISTENT_DATA_H +#ifndef BASE_RUN_SET_H +#define BASE_RUN_SET_H -#include "persistent-data/run.h" +#include "base/run.h" #include #include @@ -92,6 +92,14 @@ namespace base { return runs_.end(); } + const_iterator lower_bound(T const &b) const { + return runs_.lower_bound(run(b, b + 1)); + } + + const_iterator upper_bound(T const &b) const { + return runs_.upper_bound(run(b, b + 1)); + } + void negate() { rset replacement; diff --git a/base/sequence_generator.cc b/base/sequence_generator.cc new file mode 100644 index 0000000..cd0e3f9 --- /dev/null +++ b/base/sequence_generator.cc @@ -0,0 +1,195 @@ +#include "base/run_set.h" +#include "base/sequence_generator.h" +#include + +//---------------------------------------------------------------- + +namespace { + // - The maximum generated value is not greater than (begin+size) + // - The generated values are not aligned to the step, if the begin + // value is not aligned. + class forward_sequence_generator: public base::sequence_generator { + public: + forward_sequence_generator(uint64_t begin, + uint64_t size, + uint64_t step) + : begin_(begin), + step_(step), + current_(begin), + rounds_(0) + { + verify_parameters(size, step); + end_ = begin + (size / step) * step; + } + + forward_sequence_generator() + : begin_(0), end_(1), step_(1), + current_(0), rounds_(0) { + } + + uint64_t next() { + uint64_t r = current_; + current_ += step_; + if (current_ >= end_) { + current_ = begin_; + ++rounds_; + } + return r; + } + + void reset(uint64_t begin, uint64_t size, uint64_t step = 1) { + verify_parameters(size, step); + + begin_ = begin; + end_ = begin + (size / step) * step; + step_ = step; + current_ = begin; + rounds_ = 0; + } + + static void verify_parameters(uint64_t size, uint64_t step) { + if (!size || !step) + throw std::runtime_error("size or step must be non-zero"); + + if (size < step) + throw std::runtime_error("size must be greater than the step"); + } + + uint64_t get_rounds() { + return rounds_; + } + + private: + uint64_t begin_; + uint64_t end_; + uint64_t step_; + uint64_t current_; + uint64_t rounds_; + }; + + // - The maximum generated value is not greater than (begin+size) + // - The generated values are not aligned to the step, if the begin + // value is not aligned. + class random_sequence_generator: public base::sequence_generator { + public: + // TODO: load random seeds + random_sequence_generator(uint64_t begin, + uint64_t size, + uint64_t step, + unsigned seq_nr = 1) + : begin_(begin), + step_(step), + max_forward_steps_(seq_nr), + nr_generated_(0) + { + if (!size || !step || !seq_nr) + throw std::runtime_error("size, step, or forward steps must be non-zero"); + + if (size < step) + throw std::runtime_error("size must be greater than the step"); + + nr_steps_ = size / step; + + if (!max_forward_steps_ || max_forward_steps_ > nr_steps_) + throw std::runtime_error("invalid number of forward steps"); + + if (max_forward_steps_ > 1) + reset_forward_generator(); + } + + uint64_t next() { + // FIXME: eliminate if-else + uint64_t step_idx = (max_forward_steps_ > 1) ? + next_forward_step() : next_random_step(); + + return begin_ + step_idx * step_; + } + + private: + void reset_forward_generator() { + uint64_t begin = peek_random_step(); + + unsigned seq_nr = (std::rand() % max_forward_steps_) + 1; + base::run_set::const_iterator it = rand_map_.upper_bound(begin); + if (it != rand_map_.end()) + seq_nr = std::min(seq_nr, *it->begin_ - begin); + else + seq_nr = std::min(seq_nr, nr_steps_ - begin); + + forward_gen_.reset(begin, seq_nr); + } + + uint64_t next_forward_step() { + uint64_t step_idx = forward_gen_.next(); + consume_random_map(step_idx); + + if (forward_gen_.get_rounds()) + reset_forward_generator(); + + return step_idx; + } + + uint64_t next_random_step() { + uint64_t step_idx = peek_random_step(); + consume_random_map(step_idx); + + return step_idx; + } + + void consume_random_map(uint64_t step_idx) { + rand_map_.add(step_idx); + + ++nr_generated_; + + // wrap-around + if (nr_generated_ == nr_steps_) { + rand_map_.clear(); + nr_generated_ = 0; + } + } + + uint64_t peek_random_step() const { + uint64_t step_idx; + + bool found = true; + while (found) { + step_idx = std::rand() % nr_steps_; + found = rand_map_.member(step_idx); + } + + return step_idx; + } + + uint64_t begin_; + uint64_t nr_steps_; + uint64_t step_; + unsigned max_forward_steps_; + + base::run_set rand_map_; + uint64_t nr_generated_; + forward_sequence_generator forward_gen_; + }; +} + +//---------------------------------------------------------------- + +base::sequence_generator::ptr +base::create_forward_sequence_generator(uint64_t begin, + uint64_t size, + uint64_t step) +{ + return sequence_generator::ptr( + new forward_sequence_generator(begin, size, step)); +} + +base::sequence_generator::ptr +base::create_random_sequence_generator(uint64_t begin, + uint64_t size, + uint64_t step, + unsigned seq_nr) +{ + return sequence_generator::ptr( + new random_sequence_generator(begin, size, step, seq_nr)); +} + +//---------------------------------------------------------------- diff --git a/base/sequence_generator.h b/base/sequence_generator.h new file mode 100644 index 0000000..fa2150a --- /dev/null +++ b/base/sequence_generator.h @@ -0,0 +1,27 @@ +#ifndef BASE_SEQUENCE_GENERATOR_H +#define BASE_SEQUENCE_GENERATOR_H + +#include + +//---------------------------------------------------------------- + +namespace base { + class sequence_generator { + public: + typedef std::shared_ptr ptr; + + virtual uint64_t next() = 0; + }; + + sequence_generator::ptr + create_forward_sequence_generator(uint64_t begin, uint64_t size, + uint64_t step); + + sequence_generator::ptr + create_random_sequence_generator(uint64_t begin, uint64_t size, + uint64_t step, unsigned seq_nr = 1); +} + +//---------------------------------------------------------------- + +#endif diff --git a/persistent-data/block_counter.h b/persistent-data/block_counter.h index 0b7ba5b..6fc885c 100644 --- a/persistent-data/block_counter.h +++ b/persistent-data/block_counter.h @@ -19,8 +19,8 @@ #ifndef BLOCK_COUNTER_H #define BLOCK_COUNTER_H +#include "base/run_set.h" #include "block.h" -#include "run_set.h" //---------------------------------------------------------------- diff --git a/persistent-data/data-structures/bitset.h b/persistent-data/data-structures/bitset.h index ca98a86..e540237 100644 --- a/persistent-data/data-structures/bitset.h +++ b/persistent-data/data-structures/bitset.h @@ -19,7 +19,7 @@ #ifndef PERSISTENT_DATA_DATA_STRUCTURES_BITSET_H #define PERSISTENT_DATA_DATA_STRUCTURES_BITSET_H -#include "persistent-data/run.h" +#include "base/run.h" //---------------------------------------------------------------- diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 7eaa8d2..2c8e28c 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -1,9 +1,9 @@ #ifndef PERSISTENT_DATA_DATA_STRUCTURES_DAMAGE_VISITOR_H #define PERSISTENT_DATA_DATA_STRUCTURES_DAMAGE_VISITOR_H +#include "base/run.h" #include "persistent-data/data-structures/btree.h" #include "persistent-data/data-structures/btree_node_checker.h" -#include "persistent-data/run.h" //---------------------------------------------------------------- diff --git a/persistent-data/space-maps/subtracting_span_iterator.h b/persistent-data/space-maps/subtracting_span_iterator.h index 8d83188..6a803ad 100644 --- a/persistent-data/space-maps/subtracting_span_iterator.h +++ b/persistent-data/space-maps/subtracting_span_iterator.h @@ -1,8 +1,8 @@ #ifndef SPAN_ITERATOR_H #define SPAN_ITERATOR_H +#include "base/run_set.h" #include "persistent-data/space_map.h" -#include "persistent-data/run_set.h" #include diff --git a/persistent-data/space_map.h b/persistent-data/space_map.h index 15b6076..3584d59 100644 --- a/persistent-data/space_map.h +++ b/persistent-data/space_map.h @@ -19,9 +19,9 @@ #ifndef SPACE_MAP_H #define SPACE_MAP_H +#include "base/run.h" #include "persistent-data/block.h" #include "persistent-data/block_counter.h" -#include "persistent-data/run.h" #include #include diff --git a/thin-provisioning/device_tree.h b/thin-provisioning/device_tree.h index 5bfefda..ee241a5 100644 --- a/thin-provisioning/device_tree.h +++ b/thin-provisioning/device_tree.h @@ -1,8 +1,8 @@ #ifndef THIN_DEVICE_CHECKER_H #define THIN_DEVICE_CHECKER_H +#include "base/run.h" #include "persistent-data/data-structures/btree.h" -#include "persistent-data/run.h" //---------------------------------------------------------------- diff --git a/thin-provisioning/mapping_tree.h b/thin-provisioning/mapping_tree.h index 7545b74..870d614 100644 --- a/thin-provisioning/mapping_tree.h +++ b/thin-provisioning/mapping_tree.h @@ -1,8 +1,8 @@ #ifndef MAPPING_TREE_H #define MAPPING_TREE_H +#include "base/run.h" #include "persistent-data/data-structures/btree.h" -#include "persistent-data/run.h" //---------------------------------------------------------------- diff --git a/thin-provisioning/rmap_visitor.h b/thin-provisioning/rmap_visitor.h index 98fb9fc..d9cb97d 100644 --- a/thin-provisioning/rmap_visitor.h +++ b/thin-provisioning/rmap_visitor.h @@ -1,7 +1,7 @@ #ifndef THIN_RMAP_VISITOR_H #define THIN_RMAP_VISITOR_H -#include "persistent-data/run.h" +#include "base/run.h" #include "thin-provisioning/mapping_tree.h" //---------------------------------------------------------------- diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index b1055d1..5b31608 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -7,8 +7,8 @@ #include "version.h" #include "base/indented_stream.h" +#include "base/run.h" #include "persistent-data/data-structures/btree_damage_visitor.h" -#include "persistent-data/run.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/space-maps/disk.h" #include "persistent-data/file_utils.h" diff --git a/thin-provisioning/thin_rmap.cc b/thin-provisioning/thin_rmap.cc index 8f73c60..6782c77 100644 --- a/thin-provisioning/thin_rmap.cc +++ b/thin-provisioning/thin_rmap.cc @@ -6,8 +6,8 @@ #include "version.h" +#include "base/run.h" #include "persistent-data/data-structures/btree_damage_visitor.h" -#include "persistent-data/run.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/file_utils.h" #include "thin-provisioning/commands.h" diff --git a/unit-tests/Makefile.in b/unit-tests/Makefile.in index 84c8d1d..a8f1c8c 100644 --- a/unit-tests/Makefile.in +++ b/unit-tests/Makefile.in @@ -65,6 +65,7 @@ TEST_SOURCE=\ unit-tests/rmap_visitor_t.cc \ unit-tests/rolling_hash_t.cc \ unit-tests/run_set_t.cc \ + unit-tests/sequence_generator_t.cc \ unit-tests/space_map_t.cc \ unit-tests/span_iterator_t.cc \ unit-tests/transaction_manager_t.cc diff --git a/unit-tests/btree_damage_visitor_t.cc b/unit-tests/btree_damage_visitor_t.cc index 25855f7..c5590fa 100644 --- a/unit-tests/btree_damage_visitor_t.cc +++ b/unit-tests/btree_damage_visitor_t.cc @@ -3,10 +3,10 @@ #include "test_utils.h" #include "base/endian_utils.h" +#include "base/run.h" #include "persistent-data/data-structures/btree_damage_visitor.h" #include "persistent-data/space-maps/core.h" #include "persistent-data/transaction_manager.h" -#include "persistent-data/run.h" using namespace base; using namespace std; diff --git a/unit-tests/run_set_t.cc b/unit-tests/run_set_t.cc index 53d1bc1..33c2b02 100644 --- a/unit-tests/run_set_t.cc +++ b/unit-tests/run_set_t.cc @@ -1,6 +1,6 @@ #include "gmock/gmock.h" -#include "persistent-data/run_set.h" +#include "base/run_set.h" #include diff --git a/unit-tests/sequence_generator_t.cc b/unit-tests/sequence_generator_t.cc new file mode 100644 index 0000000..3f13c19 --- /dev/null +++ b/unit-tests/sequence_generator_t.cc @@ -0,0 +1,208 @@ +#include "gmock/gmock.h" + +#include "base/sequence_generator.h" + +using namespace base; +using namespace std; +using namespace testing; + +//---------------------------------------------------------------- + +namespace { + class SequenceGeneratorTests : public Test { + }; + + class ForwardSequenceGeneratorTests : public Test { + public: + ForwardSequenceGeneratorTests() { + } + + void reset(uint64_t begin, uint64_t size, uint64_t step) { + gen_ = create_forward_sequence_generator( + begin, size, step); + + // the maximum value is expected to be less than + // (begin + round_down(size, step)) + begin_ = begin; + end_ = begin + (size / step) * step; + step_ = step; + } + + void test_full_sequence() { + for (uint64_t i = begin_; i < end_; i += step_) + ASSERT_EQ(i, gen_->next()); + } + + private: + sequence_generator::ptr gen_; + uint64_t begin_; + uint64_t end_; + uint64_t step_; + }; + + class RandomSequenceGeneratorTests : public Test { + struct span_tracker { + span_tracker(uint64_t begin, uint64_t step) + : last_(begin), + step_(step), + len_(1), + max_len_(1) { + } + + void append(uint64_t n) { + if (n == last_ + step_) + ++len_; + else + len_ = 1; + + if (len_ > max_len_) + max_len_ = len_; + + last_ = n; + } + + uint64_t last_; + uint64_t step_; + unsigned len_; + unsigned max_len_; + }; + + public: + RandomSequenceGeneratorTests() { + } + + void reset(uint64_t begin, uint64_t size, uint64_t step, + unsigned seq_nr = 1) { + gen_ = create_random_sequence_generator( + begin, size, step, seq_nr); + + // the maximum value is expected to be less than + // (begin + round_down(size, step)) + begin_ = begin; + nr_steps_ = size / step; + step_ = step; + seq_nr_ = seq_nr; + } + + void test_full_sequence() { + std::set values; + + // generate the whole sequence + uint64_t n = gen_->next(); + values.insert(n); + span_tracker tracker(n, step_); + + for (uint64_t i = 1; i < nr_steps_; i++) { + n = gen_->next(); + values.insert(n); + tracker.append(n); + } + + // verify the generated sequence + ASSERT_EQ(values.size(), nr_steps_); // assert no duplicates + + set::const_iterator it = values.begin(); + uint64_t end = begin_ + nr_steps_ * step_; + for (uint64_t v = begin_; v < end; v += step_) { + ASSERT_EQ(v, *it); + ++it; + } + + // FIXME: refine this assertion + // The maximum span length depends on the randomness. + // It should be greater than half of the desired length, + // if the probabilities are evenly distributed. + if (seq_nr_ > 1) { + ASSERT_TRUE(tracker.max_len_ >= seq_nr_ / 2); + } + } + + private: + sequence_generator::ptr gen_; + uint64_t begin_; + uint64_t nr_steps_; + uint64_t step_; + unsigned seq_nr_; + }; +} + +//---------------------------------------------------------------- + +TEST_F(SequenceGeneratorTests, create_with_valid_params) +{ + // step < size + create_forward_sequence_generator(0, 10, 2); + + // boundary test: step == size + create_forward_sequence_generator(0, 10, 10); + + // step < size + create_random_sequence_generator(0, 10, 2); + + // boundary test: step == size + create_random_sequence_generator(0, 10, 10); + + // seq_nr < nr_steps + create_random_sequence_generator(0, 10, 2, 2); + + // boundary test: seq_nr == nr_steps + create_random_sequence_generator(0, 10, 2, 5); +} + +TEST_F(SequenceGeneratorTests, create_with_invalid_params) +{ + // zero size or step + ASSERT_THROW(create_forward_sequence_generator(0, 1, 0), std::runtime_error); + ASSERT_THROW(create_forward_sequence_generator(0, 0, 1), std::runtime_error); + + // step > size + ASSERT_THROW(create_forward_sequence_generator(0, 10, 11), std::runtime_error); + + // zero size, step, or seq_nr + ASSERT_THROW(create_random_sequence_generator(0, 1, 0), std::runtime_error); + ASSERT_THROW(create_random_sequence_generator(0, 0, 1), std::runtime_error); + ASSERT_THROW(create_random_sequence_generator(0, 1, 1, 0), std::runtime_error); + + // step > size + ASSERT_THROW(create_random_sequence_generator(0, 10, 11), std::runtime_error); + + // seq_nr > nr_steps + ASSERT_THROW(create_random_sequence_generator(0, 10, 2, 6), std::runtime_error); +} + +TEST_F(ForwardSequenceGeneratorTests, forward_sequence_aligned) +{ + reset(50, 100, 2); // begin & size are aligned to step + test_full_sequence(); + test_full_sequence(); // wrap-around +} + +TEST_F(ForwardSequenceGeneratorTests, forward_sequence_unaligned) +{ + reset(50, 100, 7); // begin & size are not aligned to step + test_full_sequence(); + test_full_sequence(); // wrap-around +} + +TEST_F(RandomSequenceGeneratorTests, random_sequence_aligned) +{ + reset(50, 100, 2); // begin & size are aligned to step + test_full_sequence(); + test_full_sequence(); // wrap-around +} + +TEST_F(RandomSequenceGeneratorTests, random_sequence_unaligned) +{ + reset(50, 100, 7); // begin & size are not aligned to step + test_full_sequence(); + test_full_sequence(); // wrap-around +} + +TEST_F(RandomSequenceGeneratorTests, random_sequence_with_span) +{ + reset(50, 10000, 2, 10); // begin & size are aligned to step + test_full_sequence(); + test_full_sequence(); // wrap-around +} + +//---------------------------------------------------------------- diff --git a/unit-tests/span_iterator_t.cc b/unit-tests/span_iterator_t.cc index c0771d8..76ff70e 100644 --- a/unit-tests/span_iterator_t.cc +++ b/unit-tests/span_iterator_t.cc @@ -1,7 +1,7 @@ #include "gmock/gmock.h" +#include "base/run_set.h" #include "persistent-data/space-maps/subtracting_span_iterator.h" -#include "persistent-data/run_set.h" using namespace std; using namespace persistent_data;