From 0e1700fbe9e34c58f1019a0c425fad9e62db24a3 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 22 May 2020 14:11:48 +0100 Subject: [PATCH] [thin_metadata_pack] First pass at pack/unpack --- Makefile.in | 9 +- bin/thin_metadata_pack | 1 + bin/thin_metadata_unpack | 1 + man8/thin_metadata_pack.txt | 26 ++ man8/thin_metadata_unpack.txt | 28 ++ thin-provisioning/commands.cc | 2 + thin-provisioning/commands.h | 16 ++ thin-provisioning/thin_metadata_pack.cc | 328 ++++++++++++++++++++++++ 8 files changed, 410 insertions(+), 1 deletion(-) create mode 120000 bin/thin_metadata_pack create mode 120000 bin/thin_metadata_unpack create mode 100644 man8/thin_metadata_pack.txt create mode 100644 man8/thin_metadata_unpack.txt create mode 100644 thin-provisioning/thin_metadata_pack.cc diff --git a/Makefile.in b/Makefile.in index afaaa14..1628533 100644 --- a/Makefile.in +++ b/Makefile.in @@ -113,6 +113,7 @@ SOURCE=\ thin-provisioning/thin_dump.cc \ thin-provisioning/thin_ls.cc \ thin-provisioning/thin_metadata_size.cc \ + thin-provisioning/thin_metadata_pack.cc \ thin-provisioning/thin_pool.cc \ thin-provisioning/thin_repair.cc \ thin-provisioning/thin_restore.cc \ @@ -166,7 +167,7 @@ CXXFLAGS+=@CXXDEBUG_FLAG@ CXXFLAGS+=@CXX_STRERROR_FLAG@ CXXFLAGS+=@LFS_FLAGS@ INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning -LIBS:=-laio -lexpat -ldl +LIBS:=-laio -lexpat -lz -lboost_iostreams -ldl ifeq ("@DEVTOOLS@", "yes") LIBS+=-lncurses @@ -271,6 +272,8 @@ TOOLS:=\ thin_repair \ thin_restore \ thin_rmap \ + thin_metadata_pack \ + thin_metadata_unpack \ thin_metadata_size \ thin_trim \ era_check \ @@ -297,6 +300,8 @@ install: bin/pdata_tools $(MANPAGES) ln -s -f pdata_tools $(BINDIR)/thin_repair ln -s -f pdata_tools $(BINDIR)/thin_restore ln -s -f pdata_tools $(BINDIR)/thin_rmap + ln -s -f pdata_tools $(BINDIR)/thin_metadata_pack + ln -s -f pdata_tools $(BINDIR)/thin_metadata_unpack ln -s -f pdata_tools $(BINDIR)/thin_metadata_size ln -s -f pdata_tools $(BINDIR)/thin_trim ln -s -f pdata_tools $(BINDIR)/era_check @@ -317,6 +322,8 @@ install: bin/pdata_tools $(MANPAGES) $(INSTALL_DATA) man8/thin_repair.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_restore.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_rmap.8 $(MANPATH)/man8 + $(INSTALL_DATA) man8/thin_metadata_pack.8 $(MANPATH)/man8 + $(INSTALL_DATA) man8/thin_metadata_unpack.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_metadata_size.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/era_check.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/era_dump.8 $(MANPATH)/man8 diff --git a/bin/thin_metadata_pack b/bin/thin_metadata_pack new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_metadata_pack @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file diff --git a/bin/thin_metadata_unpack b/bin/thin_metadata_unpack new file mode 120000 index 0000000..84c01e7 --- /dev/null +++ b/bin/thin_metadata_unpack @@ -0,0 +1 @@ +pdata_tools \ No newline at end of file diff --git a/man8/thin_metadata_pack.txt b/man8/thin_metadata_pack.txt new file mode 100644 index 0000000..d049696 --- /dev/null +++ b/man8/thin_metadata_pack.txt @@ -0,0 +1,26 @@ +NAME + thin_metadata_pack - pack thin provisioning binary metadata. + +SYNOPSIS + thin_metadata_pack [options] -i {device|file} -o {device|file} + +DESCRIPTION + thin_metadata_pack and thin_metadata_unpack are used to compress + binary metadata. Useful for support. + + thin_metadata_pack compresses the metadata, omitting any metadata blocks that are unused. + + This tool cannot be run on live metadata. + +OPTIONS + -h, --help Print help and exit. + -V, --version Print version information and exit. + -i, --input {device|file} Input file or device with binary data. + -o, --output {device|file} Output file or device for binary data. + +SEE ALSO + thin_dump(8), thin_check(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) + +AUTHOR + Joe Thornber + diff --git a/man8/thin_metadata_unpack.txt b/man8/thin_metadata_unpack.txt new file mode 100644 index 0000000..addfa66 --- /dev/null +++ b/man8/thin_metadata_unpack.txt @@ -0,0 +1,28 @@ +NAME + thin_metadata_unpack - unpack thin provisioning binary metadata. + +SYNOPSIS + thin_metadata_unpack [options] -i {device|file} -o {device|file} + +DESCRIPTION + thin_metadata_pack and thin_metadata_unpack are used to compress + binary metadata. Useful for support. + + thin_metadata_unpack expands metadata that has previously been packed with + thin_metadata_pack. It outputs a binary file that the rest of the thin + tools can use. + + This tool cannot be run on live metadata. + +OPTIONS + -h, --help Print help and exit. + -V, --version Print version information and exit. + -i, --input {device|file} Input file or device with binary data. + -o, --output {device|file} Output file or device for binary data. + +SEE ALSO + thin_dump(8), thin_check(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) + +AUTHOR + Joe Thornber + diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index 13dc76c..9732ad5 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -12,6 +12,8 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_delta_cmd())); app.add_cmd(command::ptr(new thin_dump_cmd())); app.add_cmd(command::ptr(new thin_ls_cmd())); + app.add_cmd(command::ptr(new thin_metadata_pack_cmd())); + app.add_cmd(command::ptr(new thin_metadata_unpack_cmd())); app.add_cmd(command::ptr(new thin_metadata_size_cmd())); app.add_cmd(command::ptr(new thin_restore_cmd())); app.add_cmd(command::ptr(new thin_repair_cmd())); diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index 6b80bdb..94539d1 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -71,6 +71,22 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_metadata_pack_cmd : public base::command { + public: + thin_metadata_pack_cmd(); + + virtual void usage(std::ostream &out) const override; + virtual int run(int argc, char **argv) override; + }; + + class thin_metadata_unpack_cmd : public base::command { + public: + thin_metadata_unpack_cmd(); + + virtual void usage(std::ostream &out) const override; + virtual int run(int argc, char **argv) override; + }; + #ifdef DEV_TOOLS class thin_ll_dump_cmd : public base::command { public: diff --git a/thin-provisioning/thin_metadata_pack.cc b/thin-provisioning/thin_metadata_pack.cc new file mode 100644 index 0000000..0f37ebd --- /dev/null +++ b/thin-provisioning/thin_metadata_pack.cc @@ -0,0 +1,328 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/disk.h" +#include "persistent-data/validators.h" +#include "thin-provisioning/commands.h" +#include "thin-provisioning/superblock.h" +#include "version.h" + +using namespace thin_provisioning; +using namespace persistent_data; + +using boost::optional; + +//--------------------------------------------------------------------------- + +namespace { + using namespace std; + constexpr uint64_t MAGIC = 0xa537a0aa6309ef77; + + // Pack file format + // ---------------- + // + // file := * + // file-header := MAGIC BLOCK_SIZE NR_BLOCKS NR_ENTRIES + // entry := BLOCK_NR BYTES class flags { + + struct flags { + optional input_file_; + optional output_file_; + }; + + class is_metadata_functor { + public: + is_metadata_functor() + : superblock_v_(superblock_validator()), + btree_node_v_(create_btree_node_validator()), + bitmap_v_(bitmap_validator()), + index_v_(index_validator()) { + } + + bool operator() (void const *raw) const { + return superblock_v_->check_raw(raw) || + btree_node_v_->check_raw(raw) || + bitmap_v_->check_raw(raw) || + index_v_->check_raw(raw); + } + + private: + bcache::validator::ptr superblock_v_; + bcache::validator::ptr btree_node_v_; + bcache::validator::ptr bitmap_v_; + bcache::validator::ptr index_v_; + }; + + void prealloc_file(string const &file, off_t len) { + int fd = ::open(file.c_str(), O_TRUNC | O_CREAT | O_RDWR, 0666); + if (fd < 0) + throw runtime_error("couldn't open output file"); + + if (::fallocate(fd, 0, 0, len)) + throw runtime_error("couldn't fallocate"); + ::close(fd); + } + + uint64_t read_u64(istream &in) { + base::le64 n; + in.read(reinterpret_cast(&n), sizeof(n)); + + if (!in) + throw runtime_error("couldn't read u64"); + + return base::to_cpu(n); + } + + void write_u64(ostream &out, uint64_t n) { + base::le64 n_le = base::to_disk(n); + out.write(reinterpret_cast(&n_le), sizeof(n_le)); + + if (!out) + throw runtime_error("couldn't write u64"); + } + + int pack(flags const &f) { + using namespace boost::iostreams; + + std::ofstream out_file(*f.output_file_, ios_base::binary); + boost::iostreams::filtering_ostreambuf out_buf; + out_buf.push(zlib_compressor()); + out_buf.push(out_file); + std::ostream out(&out_buf); + + block_manager::ptr bm = open_bm(*f.input_file_, block_manager::READ_ONLY, true); + + uint64_t block_size = 4096; + auto nr_blocks = bm->get_nr_blocks(); + + cerr << "nr_blocks = " << nr_blocks << "\n"; + + write_u64(out, MAGIC); + write_u64(out, block_size); + write_u64(out, nr_blocks); + + is_metadata_functor is_metadata; + for (block_address b = 0; b < nr_blocks; b++) { + auto rr = bm->read_lock(b); + + if (is_metadata(rr.data())) { + write_u64(out, b); + out.write(reinterpret_cast(rr.data()), block_size); + } + } + + return 0; + } + + int unpack(flags const &f) + { + using namespace boost::iostreams; + + ifstream in_file(*f.input_file_, ios_base::binary); + if (!in_file) + throw runtime_error("couldn't open pack file"); + + filtering_istreambuf in_buf; + in_buf.push(zlib_decompressor()); + in_buf.push(in_file); + std::istream in(&in_buf); + + if (read_u64(in) != MAGIC) + throw runtime_error("not a pack file"); + + auto block_size = read_u64(in); + auto nr_blocks = read_u64(in); + + prealloc_file(*f.output_file_, nr_blocks * block_size); + block_manager bm(*f.output_file_, nr_blocks, 6, block_manager::READ_WRITE, true); + uint8_t bytes[block_size]; + while (true) { + uint64_t block_nr; + try { + block_nr = read_u64(in); + } catch (...) { + break; + } + + if (block_nr >= nr_blocks) + throw runtime_error("block nr out of bounds"); + + in.read(reinterpret_cast(bytes), block_size); + if (!in) + throw runtime_error("couldn't read data"); + + auto wr = bm.write_lock(block_nr); + memcpy(wr.data(), bytes, block_size); + } + + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_metadata_pack_cmd::thin_metadata_pack_cmd() + : command("thin_metadata_pack") +{ +} + +void +thin_metadata_pack_cmd::usage(ostream &out) const { + out << "Usage: " << get_name() << " [options]\n" + << "Options:\n" + << " {-i|--input} \n" + << " {-o|--output} \n" + << " {-h|--help}\n" + << " {-V|--version}" << endl; +} + +int +thin_metadata_pack_cmd::run(int argc, char **argv) +{ + const char shortopts[] = "hi:o:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "input", required_argument, NULL, 'i'}, + { "output", required_argument, NULL, 'o'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + flags f; + + int c; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'i': + f.input_file_ = optarg; + break; + + case 'o': + f.output_file_ = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr); + return 1; + } + } + + if (!f.input_file_) { + cerr << "No input file provided." << endl; + usage(cerr); + return 1; + } + + if (!f.output_file_) { + cerr << "No output file providied." << endl; + usage(cerr); + return 1; + } + + return pack(f); +} + +//--------------------------------------------------------------------------- + +thin_metadata_unpack_cmd::thin_metadata_unpack_cmd() + : command("thin_metadata_unpack") +{ +} + +void +thin_metadata_unpack_cmd::usage(ostream &out) const { + out << "Usage: " << get_name() << " [options]\n" + << "Options:\n" + << " {-i|--input} \n" + << " {-o|--output} \n" + << " {-h|--help}\n" + << " {-V|--version}" << endl; +} + +int +thin_metadata_unpack_cmd::run(int argc, char **argv) +{ + const char shortopts[] = "hi:o:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "input", required_argument, NULL, 'i'}, + { "output", required_argument, NULL, 'o'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + flags f; + + int c; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'i': + f.input_file_ = optarg; + break; + + case 'o': + f.output_file_ = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr); + return 1; + } + } + + if (!f.input_file_) { + cerr << "No input file provided." << endl; + usage(cerr); + return 1; + } + + if (!f.output_file_) { + cerr << "No output file providied." << endl; + usage(cerr); + return 1; + } + + return unpack(f); +} + +//---------------------------------------------------------------------------