emo
/
thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #158 from mingnus/main 

[cache_writeback] Support offset within the source and destination devices
This commit is contained in:
Joe Thornber 2021-01-20 11:06:21 +00:00 committed by GitHub
parent 76ac202463 4c17076f09
commit 9733ceb949
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 122 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
block-cache/copier.cc
View File

@ -10,12 +10,14 @@ using namespace std;
copier::copier(io_engine &engine,
string const &src, string const &dest,
sector_t block_size, size_t mem)
sector_t block_size, size_t mem,
sector_t src_offset, sector_t dest_offset)
: pool_(block_size * 512, mem, PAGE_SIZE),
block_size_(block_size),
engine_(engine),
src_handle_(engine_.open_file(src, io_engine::M_READ_ONLY)),
dest_handle_(engine_.open_file(dest, io_engine::M_READ_WRITE)),
src_offset_(src_offset), dest_offset_(dest_offset),
genkey_count_(0)
{
}
@ -45,8 +47,8 @@ copier::issue(copy_op const &op)
auto r = engine_.issue_io(src_handle_,
io_engine::D_READ,
to_sector(op.src_b),
to_sector(op.src_e),
to_src_sector(op.src_b),
to_src_sector(op.src_e),
data,
key);
@ -151,8 +153,8 @@ copier::wait_successful(io_engine::wait_result const &p)
j.op.read_complete = true;
if (!engine_.issue_io(dest_handle_,
io_engine::D_WRITE,
to_sector(j.op.dest_b),
to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)),
to_dest_sector(j.op.dest_b),
to_dest_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)),
j.data,
it->first)) {
complete(j);
@ -177,9 +179,15 @@ copier::complete(copy_job const &j)
}
sector_t
copier::to_sector(block_address b) const
copier::to_src_sector(block_address b) const
{
return b * block_size_;
return src_offset_ + b * block_size_;
}
sector_t
copier::to_dest_sector(block_address b) const
{
return dest_offset_ + b * block_size_;
}
unsigned

16
block-cache/copier.h
View File

@ -61,13 +61,22 @@ namespace bcache {
public:
copier(io_engine &engine,
std::string const &src, std::string const &dest,
sector_t block_size, size_t mem);
sector_t block_size, size_t mem,
sector_t src_offset, sector_t dest_offset);
~copier();
sector_t get_block_size() const {
return block_size_;
}
sector_t get_src_offset() const {
return src_offset_;
}
sector_t get_dest_offset() const {
return dest_offset_;
}
// Blocks if out of memory.
void issue(copy_op const &op);
@ -83,7 +92,8 @@ namespace bcache {
void wait_();
void complete(copy_job const &j);
sector_t to_sector(block_address b) const;
sector_t to_src_sector(block_address b) const;
sector_t to_dest_sector(block_address b) const;
unsigned genkey();
mempool pool_;
@ -91,6 +101,8 @@ namespace bcache {
io_engine &engine_;
io_engine::handle src_handle_;
io_engine::handle dest_handle_;
sector_t src_offset_;
sector_t dest_offset_;
unsigned genkey_count_;
using job_map = std::map<unsigned, copy_job>;

27
caching/cache_writeback.cc
View File

@ -31,6 +31,8 @@ namespace {
flags()
: cache_size(4 * 1024 * 1024),
sort_buffers(16 * 1024),
origin_dev_offset(0),
fast_dev_offset(0),
list_failed_blocks(false),
update_metadata(true) {
}
@ -52,6 +54,8 @@ namespace {
maybe_string metadata_dev;
maybe_string origin_dev;
maybe_string fast_dev;
sector_t origin_dev_offset;
sector_t fast_dev_offset;
bool list_failed_blocks;
bool update_metadata;
};
@ -303,7 +307,9 @@ namespace {
unsigned max_ios = f.cache_size / (md.sb_.data_block_size << SECTOR_SHIFT);
aio_engine engine(max_ios);
copier c(engine, *f.fast_dev, *f.origin_dev,
md.sb_.data_block_size, f.cache_size);
md.sb_.data_block_size, f.cache_size,
f.fast_dev_offset >> SECTOR_SHIFT,
f.origin_dev_offset >> SECTOR_SHIFT);
auto bar = create_progress_bar("Copying data");
copy_visitor cv(c, f.sort_buffers, clean_shutdown(md), f.list_failed_blocks,
@ -364,6 +370,8 @@ cache_writeback_cmd::usage(std::ostream &out) const
<< "\t\t--buffer-size-meg <size>\n"
<< "\t\t--list-failed-blocks\n"
<< "\t\t--no-metadata-update\n"
<< "\t\t--origin-device-offset <bytes>\n"
<< "\t\t--fast-device-offset <bytes>\n"
<< "Options:\n"
<< " {-h|--help}\n"
<< " {-V|--version}" << endl;
@ -382,6 +390,8 @@ cache_writeback_cmd::run(int argc, char **argv)
{ "buffer-size-meg", required_argument, NULL, 3 },
{ "list-failed-blocks", no_argument, NULL, 4 },
{ "no-metadata-update", no_argument, NULL, 5 },
{ "origin-device-offset", required_argument, NULL, 6 },
{ "fast-device-offset", required_argument, NULL, 7 },
{ "help", no_argument, NULL, 'h'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
@ -413,6 +423,14 @@ cache_writeback_cmd::run(int argc, char **argv)
fs.update_metadata = false;
break;
case 6:
fs.origin_dev_offset = parse_uint64(optarg, "origin dev offset");
break;
case 7:
fs.fast_dev_offset = parse_uint64(optarg, "fast dev offset");
break;
case 'h':
usage(cout);
return 0;
@ -452,6 +470,13 @@ cache_writeback_cmd::run(int argc, char **argv)
return 1;
}
if (fs.origin_dev_offset & (SECTOR_SHIFT - 1) ||
fs.fast_dev_offset & (SECTOR_SHIFT - 1)) {
cerr << "Offset must be sector-aligned\n\n";
usage(cerr);
return 1;
}
return writeback(fs);
}

75
unit-tests/copier_t.cc
View File

@ -59,6 +59,12 @@ namespace {
}
unique_ptr<copier> make_copier() {
return make_copier(BLOCK_SIZE, 0, 0);
}
unique_ptr<copier> make_copier(sector_t block_size,
sector_t src_offset,
sector_t dest_offset) {
EXPECT_CALL(engine_, open_file(src_file_, io_engine::M_READ_ONLY, io_engine::EXCLUSIVE)).
WillOnce(Return(SRC_HANDLE));
EXPECT_CALL(engine_, open_file(dest_file_, io_engine::M_READ_WRITE, io_engine::EXCLUSIVE)).
@ -69,20 +75,22 @@ namespace {
return unique_ptr<copier>(new copier(engine_, src_file_,
dest_file_,
BLOCK_SIZE, 1 * 1024 * 1024));
block_size, 1 * 1024 * 1024,
src_offset, dest_offset));
}
static optional<wait_result> make_wr(bool success, unsigned context) {
return optional<wait_result>(wait_result(success, context));
}
// issue a copy_op, and wait for its completion synchronously
void issue_successful_op(copier &c, copy_op &op, unsigned context) {
InSequence dummy;
unsigned nr_pending = c.nr_pending();
EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ,
op.src_b * BLOCK_SIZE,
op.src_e * BLOCK_SIZE, _, context)).
expected_src_sector(c, op.src_b),
expected_src_sector(c, op.src_e), _, context)).
WillOnce(Return(true));
c.issue(op);
@ -92,8 +100,9 @@ namespace {
WillOnce(Return(make_wr(true, context)));
EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE,
op.dest_b * BLOCK_SIZE,
(op.dest_b + (op.src_e - op.src_b)) * BLOCK_SIZE, _, context)).
expected_dest_sector(c, op.dest_b),
expected_dest_sector(c, op.dest_b + (op.src_e - op.src_b)),
_, context)).
WillOnce(Return(true));
EXPECT_CALL(engine_, wait()).
@ -108,28 +117,63 @@ namespace {
unsigned const SRC_HANDLE = 10;
unsigned const DEST_HANDLE = 11;
StrictMock<io_engine_mock> engine_;
private:
sector_t expected_src_sector(const copier &c, block_address b) {
return c.get_src_offset() + b * c.get_block_size();
}
sector_t expected_dest_sector(const copier &c, block_address b) {
return c.get_dest_offset() + b * c.get_block_size();
}
string src_file_;
string dest_file_;
StrictMock<io_engine_mock> engine_;
};
}
//----------------------------------------------------------------
TEST_F(CopierTests, empty_test)
TEST_F(CopierTests, create_default_copier)
{
auto c = make_copier();
ASSERT_EQ(c->get_block_size(), BLOCK_SIZE);
ASSERT_EQ(c->get_src_offset(), 0u);
ASSERT_EQ(c->get_dest_offset(), 0u);
}
TEST_F(CopierTests, create_copier_with_offsets)
{
sector_t src_offset = 2048;
sector_t dest_offset = 8192;
auto c = make_copier(BLOCK_SIZE, src_offset, dest_offset);
ASSERT_EQ(c->get_block_size(), BLOCK_SIZE);
ASSERT_EQ(c->get_src_offset(), src_offset);
ASSERT_EQ(c->get_dest_offset(), dest_offset);
}
TEST_F(CopierTests, successful_copy)
{
// Copy first block
// Copy first block
copy_op op1(0, 1, 0);
auto c = make_copier();
issue_successful_op(*c, op1, 0);
}
TEST_F(CopierTests, successful_copy_with_offsets)
{
copy_op op1(0, 1, 0);
auto c = make_copier(BLOCK_SIZE, 2048, 8192);
issue_successful_op(*c, op1, 0);
}
// Verify copier's error handling against unsucessful engine operations
// at different stages.
// Test the first stage (issue_read (failed) => read => issue_write => write)
TEST_F(CopierTests, unsuccessful_issue_read)
{
copy_op op1(0, 1, 0);
@ -147,6 +191,7 @@ TEST_F(CopierTests, unsuccessful_issue_read)
ASSERT_FALSE(mop->success());
}
// Test the second stage (issue_read => read (failed) => issue_write => write)
TEST_F(CopierTests, unsuccessful_read)
{
copy_op op1(0, 1, 0);
@ -167,6 +212,7 @@ TEST_F(CopierTests, unsuccessful_read)
ASSERT_FALSE(mop->success());
}
// Test the third stage (issue_read => read => issue_write (failed) => write)
TEST_F(CopierTests, unsuccessful_issue_write)
{
copy_op op1(0, 1, 0);
@ -191,6 +237,7 @@ TEST_F(CopierTests, unsuccessful_issue_write)
ASSERT_FALSE(mop->success());
}
// Test the last stage (issue_read => read => issue_write => write (failed))
TEST_F(CopierTests, unsuccessful_write)
{
// Copy first block
@ -237,6 +284,18 @@ TEST_F(CopierTests, copy_different_blocks)
}
}
TEST_F(CopierTests, copy_different_blocks_with_offsets)
{
sector_t src_offset = 2048;
sector_t dest_offset = 8192;
auto c = make_copier(BLOCK_SIZE, src_offset, dest_offset);
for (unsigned i = 0; i < 5000; i++) {
copy_op op(i, i + 1, i);
issue_successful_op(*c, op, i);
}
}
TEST_F(CopierTests, wait_can_timeout)
{
copy_op op1(0, 1, 0);