61de3f9287
The Rust implementation is multithreaded, performs better in general and
does custom compression of btree nodes to achieve much better compression
ratios. unpack also checksums expanded metadata to validate it.
Format version has jumped to 3, no backwards compatibility, but I think
that's ok since we never made a release that contained the C++ version
of these tools.
Benchmarks
==========
On an 8 core, 16 hyperthread machine.
metadata 1G, full:
Pack size pack time unpack time
------------------------------------------------------
C++ 193M 50.3s 6.9s (no verify)
Rust 70M 1.4s 1.8s (verify)
metadata 16G, sparse:
Pack size pack time unpack time
------------------------------------------------------
C++ 21M 68s 1s (no verify)
Rust 4M 8.6s 0.5s (verify)
52 lines
1.2 KiB
Rust
52 lines
1.2 KiB
Rust
use std::io;
|
|
use std::io::{Read, Seek};
|
|
use std::fs::OpenOptions;
|
|
use std::os::unix::fs::OpenOptionsExt;
|
|
use std::fs::File;
|
|
|
|
pub const BLOCK_SIZE: usize = 4096;
|
|
|
|
#[repr(align(4096))]
|
|
pub struct Block {
|
|
pub data: [u8; BLOCK_SIZE as usize],
|
|
}
|
|
|
|
pub struct BlockManager {
|
|
pub nr_blocks: u64,
|
|
input: File,
|
|
}
|
|
|
|
fn get_nr_blocks(path: &str) -> io::Result<u64> {
|
|
let metadata = std::fs::metadata(path)?;
|
|
Ok(metadata.len() / (BLOCK_SIZE as u64))
|
|
}
|
|
|
|
impl BlockManager {
|
|
pub fn new(path: &str, _cache_size: usize) -> io::Result<BlockManager> {
|
|
let input = OpenOptions::new()
|
|
.read(true)
|
|
.write(false)
|
|
.custom_flags(libc::O_DIRECT)
|
|
.open(path)?;
|
|
|
|
Ok(BlockManager {
|
|
nr_blocks: get_nr_blocks(path)?,
|
|
input: input,
|
|
})
|
|
}
|
|
|
|
pub fn get(&mut self, b: u64) -> io::Result<Block> {
|
|
self.read_block(b)
|
|
}
|
|
|
|
fn read_block(&mut self, b: u64) -> io::Result<Block>
|
|
{
|
|
let mut buf = Block {data: [0; BLOCK_SIZE]};
|
|
|
|
self.input.seek(io::SeekFrom::Start(b * (BLOCK_SIZE as u64)))?;
|
|
self.input.read_exact(&mut buf.data)?;
|
|
|
|
Ok(buf)
|
|
}
|
|
}
|