Utility/storage
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

Implement methods for block file.

Browse Source
This commit is contained in:
yukirij 2025-03-30 13:06:42 -07:00
parent 8945366082
commit adad61490b
2 changed files with 215 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/bin/test.rs
View File

@ -1,24 +1,20 @@
//use storage::BlockFile;
use storage::TrieFile;
use storage::*;
fn main()
{
std::fs::create_dir_all("data").ok();
/*if let Ok(mut bf) = BlockFile::<128>::open("data/cache_data.bin") {
for i in 0..760 {
if let Ok(id) = bf.insert(format!("Hello, world! {}. This is long text to increase the block size to sufficient length to roll over into a second block when using smaller block sizes.", i).as_bytes()) {
if let Ok(mut bf) = BlockFile::<16>::open("data/cache_data.bin") {
if let Ok(id) = bf.insert("This is a test of the block file system.".as_bytes()) {
let data = String::from_utf8(bf.get(id).unwrap()).unwrap();
println!("id {} = '{}'", id, data);
}
}
} else {
println!("Failed to open.");
}*/
}
if let Ok(mut tf) = TrieFile::<8>::open("data/cache_index.bin") {
/*if let Ok(mut tf) = TrieFile::<8>::open("data/cache_index.bin") {
for s in [
"Hello",
@ -40,5 +36,5 @@ fn main()
} else {
println!("Failed to open index.");
}
}*/
}

260
src/blockfile/mod.rs
View File

@ -70,6 +70,8 @@ impl<const Z:usize> BlockFile<Z> {
}
fn init(mut self) -> Result<Self, std::io::Error>
// Write initial headers to the file.
//
{
/*
** Header size and first two blocks are initialized for
@ -98,10 +100,12 @@ impl<const Z:usize> BlockFile<Z> {
}
pub fn insert(&mut self, data:&[u8]) -> Result<usize, std::io::Error>
// Acquire a new object id and write its data to new allocations.
//
{
// Allocate storage blocks
let block_count = (data.len() / (Z - 4)) + ((data.len() % (Z - 4)) != 0) as usize;
let block_count = Self::block_count_from_size(data.len());
//println!("block_count {}", block_count);
let blocks = self.allocate(block_count.max(1))?;
@ -112,10 +116,10 @@ impl<const Z:usize> BlockFile<Z> {
//println!("obj_id {}", id);
let mut block_data = vec![0u8; Z];
// Write data to storage blocks
let mut data_index = 0;
let mut block_data = vec![0u8; Z];
for block_index in 0..block_count {
block_data.fill(0);
@ -140,26 +144,83 @@ impl<const Z:usize> BlockFile<Z> {
Ok(id)
}
/*pub fn update(&mut self, _id:usize, _data:&[u8]) -> Result<(), std::io::Error>
pub fn update(&mut self, id:usize, data:&[u8]) -> Result<(), std::io::Error>
// Write new data to an object expanding its existing allocation.
//
{
// Get first block and data size
let (mut block_id, _) = self.get_object(id)?;
}*/
let required_blocks = Self::block_count_from_size(data.len());
let mut blocks = Vec::new();
/*pub fn remove(&mut self, _id:usize) -> Result<(), std::io::Error>
// Count blocks
while block_id != 0 {
let next = self.read_block_pointer(block_id)?;
blocks.push(block_id);
block_id = next;
}
if blocks.len() > required_blocks {
// Free excess blocks
self.release(&blocks[required_blocks..])?;
blocks.resize(required_blocks, 0);
} else if blocks.len() < required_blocks {
// Allocate additional blocks
let allocated_blocks = self.allocate(required_blocks - blocks.len())?;
blocks.extend_from_slice(&allocated_blocks);
}
// Write data to storage blocks
let mut data_index = 0;
let mut block_data = vec![0u8; Z];
for block_index in 0..blocks.len() {
block_data.fill(0);
// Copy data slice to buffer
for b in 0..Self::data_size().min(data.len() - (block_index * Self::data_size())) {
block_data[b] = data[data_index + b];
}
data_index += Self::data_size();
// Write pointer to next block to end of buffer
if block_index < blocks.len() - 1 {
let pack_next = (blocks[block_index + 1] as u32).pack();
block_data[Z - 4] = pack_next[0];
block_data[Z - 3] = pack_next[1];
block_data[Z - 2] = pack_next[2];
block_data[Z - 1] = pack_next[3];
}
self.write_block(blocks[block_index], &block_data)?;
}
Ok(())
}
pub fn remove(&mut self, _id:usize) -> Result<(), std::io::Error>
// Remove the object with the specified identifier and release its allocation.
//
{
}*/
Ok(())
}
pub fn get(&self, id:usize) -> Result<Vec<u8>, std::io::Error>
// Return the object with the specified identifier.
//
{
let mut block_data = [0u8; Z];
// Get first block and data size
let (mut block_id, size) = self.get_object(id)?;
let mut data = Vec::new();
// Read blocks until size is full
while block_id != 0 {
let block = self.read_block(block_id)?;
let next_block = u32::unpack(&block, &mut (Z - 4)).unwrap_or_default();
self.read_block(block_id, &mut block_data)?;
let next_block = u32::unpack(&block_data, &mut (Z - 4)).unwrap_or_default();
//println!("size {} len {}", size, data.len());
@ -169,7 +230,7 @@ impl<const Z:usize> BlockFile<Z> {
size - data.len()
};
data.extend_from_slice(&block[0..data_length]);
data.extend_from_slice(&block_data[0..data_length]);
block_id = next_block;
}
@ -178,6 +239,8 @@ impl<const Z:usize> BlockFile<Z> {
}
fn allocate(&mut self, count:usize) -> Result<Vec<u32>, std::io::Error>
// Mark as allocated and return the next available N blocks.
//
{
let mut b8 = [0u8; 1];
let mut b32 = [0u8; 4];
@ -213,10 +276,14 @@ impl<const Z:usize> BlockFile<Z> {
count:usize,
blocks:&mut Vec<u32>,
) -> Result<Operation, std::io::Error>
// Search allocation page tables for next available child page.
// Mark blocks allocated until requested number is acquired.
//
{
//println!("allocate_traverse()");
let mut block = self.read_block(block_id)?;
let mut block_data = [0u8; Z];
self.read_block(block_id, &mut block_data)?;
let mut write_block = false;
let mut operation = Operation::None;
@ -237,7 +304,7 @@ impl<const Z:usize> BlockFile<Z> {
cell_index = i;
let cell_byte_index = byte_index;
let cell_data = u32::unpack(&block, &mut byte_index).expect("failed to unpack during alloc");
let cell_data = u32::unpack(&block_data, &mut byte_index).expect("failed to unpack during alloc");
//println!(" - cell {}", cell_data);
@ -272,10 +339,10 @@ impl<const Z:usize> BlockFile<Z> {
// Update cell with child table reference.
let pack_child = next_block.pack();
block[cell_byte_index] = pack_child[0];
block[cell_byte_index + 1] = pack_child[1];
block[cell_byte_index + 2] = pack_child[2];
block[cell_byte_index + 3] = pack_child[3];
block_data[cell_byte_index] = pack_child[0];
block_data[cell_byte_index + 1] = pack_child[1];
block_data[cell_byte_index + 2] = pack_child[2];
block_data[cell_byte_index + 3] = pack_child[3];
write_block = true;
// If root table allocated last page, generate new root table at greater depth.
@ -300,10 +367,10 @@ impl<const Z:usize> BlockFile<Z> {
self.write_block(parent_blocks[0], &table_data)?;
// Update current table before restarting recursion.
self.write_block(block_id, &block)?;
self.write_block(block_id, &block_data)?;
// Restart recursion with new root.
self.allocate_traverse(parent_blocks[0], depth + 1, basis, true, count, blocks)?;
self.allocate_traverse(parent_blocks[0], depth + 1, 0, true, count, blocks)?;
return Ok(Operation::None);
}
}
@ -318,7 +385,7 @@ impl<const Z:usize> BlockFile<Z> {
Operation::SetOccupied => {
write_block = true;
block[cell_byte_index + 3] |= 0x80;
block_data[cell_byte_index + 3] |= 0x80;
// If last cell is marked occupied, this table is also occupied.
if cell_index == Self::table_size() {
@ -342,9 +409,9 @@ impl<const Z:usize> BlockFile<Z> {
// Find first unset bit in block.
let mut byte_index = 0;
while byte_index < block.len() && blocks.len() < count {
if block[byte_index] != 0xFF {
let bit = block[byte_index].trailing_ones();
while byte_index < block_data.len() && blocks.len() < count {
if block_data[byte_index] != 0xFF {
let bit = block_data[byte_index].trailing_ones();
let id = basis + (byte_index * 8) as u32 + bit;
//println!(" - cell {} value {:02x} bit {} alloc_id {}", byte_index, block[byte_index], bit, id);
@ -357,7 +424,7 @@ impl<const Z:usize> BlockFile<Z> {
//println!(" - basis {} byte_index {} byte {:02x} bit {} alloc {}", basis, byte_index, block[byte_index], bit, id);
// Mark block as occupied.
block[byte_index] |= 1 << bit;
block_data[byte_index] |= 1 << bit;
write_block = true;
} else {
byte_index += 1;
@ -365,23 +432,73 @@ impl<const Z:usize> BlockFile<Z> {
}
// If table is fully allocated, signal parent to mark table as not vacant.
if byte_index == block.len() && block[block.len() - 1] == 0xFF {
if byte_index == Z && block_data[Z - 1] == 0xFF {
operation = Operation::SetOccupied;
//println!("OCCUPIED!");
}
}
if write_block {
self.write_block(block_id, &block)?;
self.write_block(block_id, &block_data)?;
}
Ok(operation)
}
fn release(&mut self, blocks:&[u32]) -> Result<(),std::io::Error>
// Mark a set of blocks as unallocated.
//
{
let mut b8 = [0u8; 1];
let mut b32 = [0u8; 4];
let mut block_data = [0u8; Z];
// Read allocation table root block and depth from file.
self.file.seek(SeekFrom::Start(0))?;
self.file.read_exact(&mut b8)?;
self.file.read_exact(&mut b32)?;
let root_page = u32::unpack(&b32, &mut 0).unwrap_or_default();
let root_depth = b8[0];
for block in blocks {
let block = *block;
let mut page = root_page;
let mut depth = root_depth as u32;
let mut offset = 0;
while depth > 0 {
self.read_block(page, &mut block_data)?;
let index = (block - offset) / Self::pool_size().pow(depth as u32) as u32;
offset = Self::table_cell_offset(depth - 1, index as u32, offset) * Self::pool_size() as u32;
page = u32::unpack(&block_data, &mut (4 * index as usize)).unwrap();
depth -= 1;
}
self.read_block(page, &mut block_data)?;
let bit_index = block - offset;
let byte = bit_index / 8;
let bit = bit_index % 8;
block_data[byte as usize] &= !(1 << bit);
self.write_block(page, &block_data)?;
}
Ok(())
}
fn acquire_object(&mut self, data_id:u32, length:usize) -> Result<usize, std::io::Error>
// Allocate the next available object record.
//
{
//println!("acquire_object()");
let mut block_data = [0u8; Z];
let mut b8 = [0u8; 1];
let mut b32 = [0u8; 4];
@ -439,14 +556,14 @@ impl<const Z:usize> BlockFile<Z> {
//println!("basis: {}", basis);
let mut block = self.read_block(block_id)?;
self.read_block(block_id, &mut block_data)?;
let mut write_block = false;
let cell_index = (object_id - basis) / Self::table_offset(depth) as usize;
//println!(" - ci {}", cell_index);
let cell_start = cell_index * 4;
let cell_data = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default();
let cell_data = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default();
// Allocate new page if pointer is zero.
let child_id = if cell_data == 0 {
@ -454,10 +571,10 @@ impl<const Z:usize> BlockFile<Z> {
// Write new reference to table.
let pack_block = allocation[0].pack();
block[cell_start] = pack_block[0];
block[cell_start + 1] = pack_block[1];
block[cell_start + 2] = pack_block[2];
block[cell_start + 3] = pack_block[3];
block_data[cell_start] = pack_block[0];
block_data[cell_start + 1] = pack_block[1];
block_data[cell_start + 2] = pack_block[2];
block_data[cell_start + 3] = pack_block[3];
write_block = true;
@ -483,11 +600,11 @@ impl<const Z:usize> BlockFile<Z> {
allocation[0]
} else {
u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default()
u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default()
};
if write_block {
self.write_block(block_id, &block)?;
self.write_block(block_id, &block_data)?;
}
// Update frame of reference to child table.
@ -499,26 +616,26 @@ impl<const Z:usize> BlockFile<Z> {
//println!("end basis: {}", basis);
// Update block and header with object information.
let mut block = self.read_block(block_id)?;
self.read_block(block_id, &mut block_data)?;
let cell_index = (object_id - basis) / Self::table_offset(depth) as usize;
let cell_start = cell_index * 8;
let next_pointer = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default();
let next_pointer = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default();
//println!(" - next ptr: {}", next_pointer);
// Update cell with data location and length.
let pack_location = data_id.pack();
block[cell_start] = pack_location[0];
block[cell_start + 1] = pack_location[1];
block[cell_start + 2] = pack_location[2];
block[cell_start + 3] = pack_location[3];
block_data[cell_start] = pack_location[0];
block_data[cell_start + 1] = pack_location[1];
block_data[cell_start + 2] = pack_location[2];
block_data[cell_start + 3] = pack_location[3];
let pack_length = (length as u32).pack();
block[cell_start + 4] = pack_length[0];
block[cell_start + 5] = pack_length[1];
block[cell_start + 6] = pack_length[2];
block[cell_start + 7] = pack_length[3];
block_data[cell_start + 4] = pack_length[0];
block_data[cell_start + 5] = pack_length[1];
block_data[cell_start + 6] = pack_length[2];
block_data[cell_start + 7] = pack_length[3];
// Update header with new pointer.
let pack_pointer = next_pointer.pack();
@ -526,13 +643,17 @@ impl<const Z:usize> BlockFile<Z> {
self.file.write(&pack_pointer)?;
self.write_block(block_id, &block)?;
self.write_block(block_id, &block_data)?;
Ok(object_id)
}
fn get_object(&self, id:usize) -> Result<(u32, usize), std::io::Error>
// Find initial block and data size of an object.
//
{
let mut block_data = [0u8; Z];
//println!("get_object()");
let mut file = self.file.try_clone()?;
@ -556,15 +677,15 @@ impl<const Z:usize> BlockFile<Z> {
** Select child tables containing object_id until depth is 0.
*/
let block = self.read_block(block_id)?;
self.read_block(block_id, &mut block_data)?;
let cell_index = (id - basis) / Self::table_offset(depth) as usize;
let cell_start = cell_index * 4;
let cell_data = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default();
let cell_data = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default();
let child_id = if cell_data != 0 {
u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default()
u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default()
} else {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "object id not valid"));
};
@ -576,28 +697,44 @@ impl<const Z:usize> BlockFile<Z> {
}
// Get object pointer and length from cell.
let block = self.read_block(block_id)?;
self.read_block(block_id, &mut block_data)?;
let cell_index = (id - basis) / Self::table_offset(depth) as usize;
let mut cell_start = cell_index * 8;
let pointer = u32::unpack(&block, &mut cell_start).unwrap_or_default();
let length = u32::unpack(&block, &mut cell_start).unwrap_or_default();
let pointer = u32::unpack(&block_data, &mut cell_start).unwrap_or_default();
let length = u32::unpack(&block_data, &mut cell_start).unwrap_or_default();
Ok((pointer, length as usize))
}
fn read_block(&self, block_id:u32) -> Result<Vec<u8>, std::io::Error>
fn read_block(&self, block_id:u32, data:&mut [u8;Z]) -> Result<(),std::io::Error>
// Read a block from file.
//
{
let mut file = self.file.try_clone()?;
let mut data = vec![0u8; Z];
file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize)) as u64))?;
file.read_exact(data)?;
Ok(())
}
fn read_block_pointer(&self, block_id:u32) -> Result<u32,std::io::Error>
// Read the pointer element of a data block.
//
{
let mut file = self.file.try_clone()?;
let mut data = [0u8; 4];
file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize) + Self::data_size()) as u64))?;
file.read(&mut data)?;
Ok(data)
Ok(u32::unpack(&data, &mut 0).unwrap())
}
fn write_block(&mut self, block_id:u32, data:&[u8]) -> Result<(), std::io::Error>
// Write a block to file.
//
{
self.file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize)) as u64))?;
self.file.write(&data[0..Z])?;
@ -606,6 +743,8 @@ impl<const Z:usize> BlockFile<Z> {
}
fn end_block(&self) -> Result<u32, std::io::Error>
// Id of the last block in the file.
//
{
let mut file = self.file.try_clone()?;
let index = file.seek(SeekFrom::End(0))? as usize;
@ -622,17 +761,30 @@ impl<const Z:usize> BlockFile<Z> {
(Self::table_size() as u32).pow(depth)
}
fn block_count_from_size(size:usize) -> usize
// Number of blocks required to hold a given number of bytes.
//
{
(size / (Z - 4)) + ((size % (Z - 4)) != 0) as usize
}
const fn data_size() -> usize
// Number of bytes of data per block.
//
{
Z - 4
}
const fn table_size() -> usize
// Number of elements per table block.
//
{
Z / 8
}
const fn pool_size() -> usize
// Number of blocks represented per pool block.
//
{
Z * 8
}