use anyhow::{anyhow, Result}; use byteorder::WriteBytesExt; use std::io::Cursor; use crate::checksum; use crate::io_engine::*; use crate::math::*; use crate::pdata::array::*; use crate::pdata::btree_builder::*; use crate::pdata::unpack::*; use crate::write_batcher::*; //------------------------------------------ pub struct ArrayBlockBuilder { array_io: ArrayIO, nr_entries: u64, // size of the array entries_per_block: usize, array_blocks: Vec, // emitted array blocks values: Vec, // internal buffer } pub struct ArrayBuilder { block_builder: ArrayBlockBuilder, } struct ArrayIO { dummy: std::marker::PhantomData, } struct WriteResult { loc: u64, } //------------------------------------------ fn calc_max_entries() -> usize { (BLOCK_SIZE - ArrayBlockHeader::disk_size() as usize) / V::disk_size() as usize } //------------------------------------------ impl ArrayBlockBuilder { pub fn new(nr_entries: u64) -> ArrayBlockBuilder { let entries_per_block = calc_max_entries::(); let nr_blocks = div_up(nr_entries, entries_per_block as u64) as usize; let next_cap = std::cmp::min(nr_entries, entries_per_block as u64) as usize; ArrayBlockBuilder { array_io: ArrayIO::new(), nr_entries, entries_per_block, array_blocks: Vec::with_capacity(nr_blocks), values: Vec::::with_capacity(next_cap), } } pub fn push_value(&mut self, w: &mut WriteBatcher, index: u64, v: V) -> Result<()> { let bi = index / self.entries_per_block as u64; let i = (index % self.entries_per_block as u64) as usize; if bi < self.array_blocks.len() as u64 || i < self.values.len() || index >= self.nr_entries { return Err(anyhow!("array index out of bounds")); } while (self.array_blocks.len() as u64) < bi { self.emit_block(w)?; } if i > self.values.len() + 1 { self.values.resize_with(i - 1, Default::default); } self.values.push(v); Ok(()) } pub fn complete(mut self, w: &mut WriteBatcher) -> Result> { // Emit all the remaining queued values let nr_blocks = self.array_blocks.capacity(); while self.array_blocks.len() < nr_blocks { self.emit_block(w)?; } Ok(self.array_blocks) } /// Emit a fully utilized array block fn emit_block(&mut self, w: &mut WriteBatcher) -> Result<()> { let nr_blocks = self.array_blocks.capacity(); let cur_bi = self.array_blocks.len(); let next_cap; if cur_bi < nr_blocks - 1 { let next_begin = (cur_bi as u64 + 1) * self.entries_per_block as u64; next_cap = std::cmp::min(self.nr_entries - next_begin, self.entries_per_block as u64) as usize; } else { next_cap = 0; } let mut values = Vec::::with_capacity(next_cap); std::mem::swap(&mut self.values, &mut values); values.resize_with(values.capacity(), Default::default); let wresult = self.array_io.write(w, values)?; self.array_blocks.push(wresult.loc); Ok(()) } } //------------------------------------------ impl ArrayBuilder { pub fn new(nr_entries: u64) -> ArrayBuilder { ArrayBuilder { block_builder: ArrayBlockBuilder::::new(nr_entries), } } pub fn push_value(&mut self, w: &mut WriteBatcher, index: u64, v: V) -> Result<()> { self.block_builder.push_value(w, index, v) } pub fn complete(self, w: &mut WriteBatcher) -> Result { let blocks = self.block_builder.complete(w)?; let mut index_builder = BTreeBuilder::::new(Box::new(NoopRC {})); for (i, b) in blocks.iter().enumerate() { index_builder.push_value(w, i as u64, *b)?; } index_builder.complete(w) } } //------------------------------------------ impl ArrayIO { pub fn new() -> ArrayIO { ArrayIO { dummy: std::marker::PhantomData, } } fn write(&self, w: &mut WriteBatcher, values: Vec) -> Result { let header = ArrayBlockHeader { csum: 0, max_entries: calc_max_entries::() as u32, nr_entries: values.len() as u32, value_size: V::disk_size(), blocknr: 0, }; let ablock = ArrayBlock { header, values }; write_array_block(w, ablock) } } fn write_array_block( w: &mut WriteBatcher, mut ablock: ArrayBlock, ) -> Result { let b = w.alloc()?; ablock.set_block(b.loc); let mut cursor = Cursor::new(b.get_data()); pack_array_block(&ablock, &mut cursor)?; let loc = b.loc; w.write(b, checksum::BT::ARRAY)?; Ok(WriteResult { loc }) } fn pack_array_block( ablock: &ArrayBlock, w: &mut W, ) -> Result<()> { ablock.header.pack(w)?; for v in ablock.values.iter() { v.pack(w)?; } Ok(()) } //------------------------------------------