diff --git a/src/bin/test.rs b/src/bin/test.rs index 582fbd3..a45c278 100644 --- a/src/bin/test.rs +++ b/src/bin/test.rs @@ -1,24 +1,20 @@ //use storage::BlockFile; -use storage::TrieFile; +use storage::*; fn main() { std::fs::create_dir_all("data").ok(); - /*if let Ok(mut bf) = BlockFile::<128>::open("data/cache_data.bin") { - - for i in 0..760 { - if let Ok(id) = bf.insert(format!("Hello, world! {}. This is long text to increase the block size to sufficient length to roll over into a second block when using smaller block sizes.", i).as_bytes()) { - - let data = String::from_utf8(bf.get(id).unwrap()).unwrap(); - println!("id {} = '{}'", id, data); - } + if let Ok(mut bf) = BlockFile::<16>::open("data/cache_data.bin") { + if let Ok(id) = bf.insert("This is a test of the block file system.".as_bytes()) { + let data = String::from_utf8(bf.get(id).unwrap()).unwrap(); + println!("id {} = '{}'", id, data); } } else { println!("Failed to open."); - }*/ + } - if let Ok(mut tf) = TrieFile::<8>::open("data/cache_index.bin") { + /*if let Ok(mut tf) = TrieFile::<8>::open("data/cache_index.bin") { for s in [ "Hello", @@ -40,5 +36,5 @@ fn main() } else { println!("Failed to open index."); - } + }*/ } diff --git a/src/blockfile/mod.rs b/src/blockfile/mod.rs index 55c3d12..cfbb909 100644 --- a/src/blockfile/mod.rs +++ b/src/blockfile/mod.rs @@ -70,6 +70,8 @@ impl BlockFile { } fn init(mut self) -> Result + // Write initial headers to the file. + // { /* ** Header size and first two blocks are initialized for @@ -98,10 +100,12 @@ impl BlockFile { } pub fn insert(&mut self, data:&[u8]) -> Result + // Acquire a new object id and write its data to new allocations. + // { // Allocate storage blocks - let block_count = (data.len() / (Z - 4)) + ((data.len() % (Z - 4)) != 0) as usize; + let block_count = Self::block_count_from_size(data.len()); //println!("block_count {}", block_count); let blocks = self.allocate(block_count.max(1))?; @@ -112,10 +116,10 @@ impl BlockFile { //println!("obj_id {}", id); - let mut block_data = vec![0u8; Z]; - // Write data to storage blocks let mut data_index = 0; + let mut block_data = vec![0u8; Z]; + for block_index in 0..block_count { block_data.fill(0); @@ -140,26 +144,83 @@ impl BlockFile { Ok(id) } - /*pub fn update(&mut self, _id:usize, _data:&[u8]) -> Result<(), std::io::Error> + pub fn update(&mut self, id:usize, data:&[u8]) -> Result<(), std::io::Error> + // Write new data to an object expanding its existing allocation. + // { - - }*/ + // Get first block and data size + let (mut block_id, _) = self.get_object(id)?; - /*pub fn remove(&mut self, _id:usize) -> Result<(), std::io::Error> - { + let required_blocks = Self::block_count_from_size(data.len()); + let mut blocks = Vec::new(); + + // Count blocks + while block_id != 0 { + let next = self.read_block_pointer(block_id)?; + blocks.push(block_id); + block_id = next; + } + + if blocks.len() > required_blocks { + // Free excess blocks + self.release(&blocks[required_blocks..])?; + blocks.resize(required_blocks, 0); + + } else if blocks.len() < required_blocks { + // Allocate additional blocks + let allocated_blocks = self.allocate(required_blocks - blocks.len())?; + blocks.extend_from_slice(&allocated_blocks); + } + + // Write data to storage blocks + let mut data_index = 0; + let mut block_data = vec![0u8; Z]; - }*/ + for block_index in 0..blocks.len() { + block_data.fill(0); + + // Copy data slice to buffer + for b in 0..Self::data_size().min(data.len() - (block_index * Self::data_size())) { + block_data[b] = data[data_index + b]; + } + data_index += Self::data_size(); + + // Write pointer to next block to end of buffer + if block_index < blocks.len() - 1 { + let pack_next = (blocks[block_index + 1] as u32).pack(); + block_data[Z - 4] = pack_next[0]; + block_data[Z - 3] = pack_next[1]; + block_data[Z - 2] = pack_next[2]; + block_data[Z - 1] = pack_next[3]; + } + + self.write_block(blocks[block_index], &block_data)?; + } + + Ok(()) + } + + pub fn remove(&mut self, _id:usize) -> Result<(), std::io::Error> + // Remove the object with the specified identifier and release its allocation. + // + { + Ok(()) + } pub fn get(&self, id:usize) -> Result, std::io::Error> + // Return the object with the specified identifier. + // { + let mut block_data = [0u8; Z]; + // Get first block and data size let (mut block_id, size) = self.get_object(id)?; let mut data = Vec::new(); // Read blocks until size is full while block_id != 0 { - let block = self.read_block(block_id)?; - let next_block = u32::unpack(&block, &mut (Z - 4)).unwrap_or_default(); + self.read_block(block_id, &mut block_data)?; + let next_block = u32::unpack(&block_data, &mut (Z - 4)).unwrap_or_default(); //println!("size {} len {}", size, data.len()); @@ -169,7 +230,7 @@ impl BlockFile { size - data.len() }; - data.extend_from_slice(&block[0..data_length]); + data.extend_from_slice(&block_data[0..data_length]); block_id = next_block; } @@ -178,6 +239,8 @@ impl BlockFile { } fn allocate(&mut self, count:usize) -> Result, std::io::Error> + // Mark as allocated and return the next available N blocks. + // { let mut b8 = [0u8; 1]; let mut b32 = [0u8; 4]; @@ -213,10 +276,14 @@ impl BlockFile { count:usize, blocks:&mut Vec, ) -> Result + // Search allocation page tables for next available child page. + // Mark blocks allocated until requested number is acquired. + // { //println!("allocate_traverse()"); - let mut block = self.read_block(block_id)?; + let mut block_data = [0u8; Z]; + self.read_block(block_id, &mut block_data)?; let mut write_block = false; let mut operation = Operation::None; @@ -237,7 +304,7 @@ impl BlockFile { cell_index = i; let cell_byte_index = byte_index; - let cell_data = u32::unpack(&block, &mut byte_index).expect("failed to unpack during alloc"); + let cell_data = u32::unpack(&block_data, &mut byte_index).expect("failed to unpack during alloc"); //println!(" - cell {}", cell_data); @@ -272,10 +339,10 @@ impl BlockFile { // Update cell with child table reference. let pack_child = next_block.pack(); - block[cell_byte_index] = pack_child[0]; - block[cell_byte_index + 1] = pack_child[1]; - block[cell_byte_index + 2] = pack_child[2]; - block[cell_byte_index + 3] = pack_child[3]; + block_data[cell_byte_index] = pack_child[0]; + block_data[cell_byte_index + 1] = pack_child[1]; + block_data[cell_byte_index + 2] = pack_child[2]; + block_data[cell_byte_index + 3] = pack_child[3]; write_block = true; // If root table allocated last page, generate new root table at greater depth. @@ -300,10 +367,10 @@ impl BlockFile { self.write_block(parent_blocks[0], &table_data)?; // Update current table before restarting recursion. - self.write_block(block_id, &block)?; + self.write_block(block_id, &block_data)?; // Restart recursion with new root. - self.allocate_traverse(parent_blocks[0], depth + 1, basis, true, count, blocks)?; + self.allocate_traverse(parent_blocks[0], depth + 1, 0, true, count, blocks)?; return Ok(Operation::None); } } @@ -318,7 +385,7 @@ impl BlockFile { Operation::SetOccupied => { write_block = true; - block[cell_byte_index + 3] |= 0x80; + block_data[cell_byte_index + 3] |= 0x80; // If last cell is marked occupied, this table is also occupied. if cell_index == Self::table_size() { @@ -342,9 +409,9 @@ impl BlockFile { // Find first unset bit in block. let mut byte_index = 0; - while byte_index < block.len() && blocks.len() < count { - if block[byte_index] != 0xFF { - let bit = block[byte_index].trailing_ones(); + while byte_index < block_data.len() && blocks.len() < count { + if block_data[byte_index] != 0xFF { + let bit = block_data[byte_index].trailing_ones(); let id = basis + (byte_index * 8) as u32 + bit; //println!(" - cell {} value {:02x} bit {} alloc_id {}", byte_index, block[byte_index], bit, id); @@ -357,7 +424,7 @@ impl BlockFile { //println!(" - basis {} byte_index {} byte {:02x} bit {} alloc {}", basis, byte_index, block[byte_index], bit, id); // Mark block as occupied. - block[byte_index] |= 1 << bit; + block_data[byte_index] |= 1 << bit; write_block = true; } else { byte_index += 1; @@ -365,23 +432,73 @@ impl BlockFile { } // If table is fully allocated, signal parent to mark table as not vacant. - if byte_index == block.len() && block[block.len() - 1] == 0xFF { + if byte_index == Z && block_data[Z - 1] == 0xFF { operation = Operation::SetOccupied; //println!("OCCUPIED!"); } } if write_block { - self.write_block(block_id, &block)?; + self.write_block(block_id, &block_data)?; } Ok(operation) } + fn release(&mut self, blocks:&[u32]) -> Result<(),std::io::Error> + // Mark a set of blocks as unallocated. + // + { + let mut b8 = [0u8; 1]; + let mut b32 = [0u8; 4]; + + let mut block_data = [0u8; Z]; + + // Read allocation table root block and depth from file. + self.file.seek(SeekFrom::Start(0))?; + self.file.read_exact(&mut b8)?; + self.file.read_exact(&mut b32)?; + + let root_page = u32::unpack(&b32, &mut 0).unwrap_or_default(); + let root_depth = b8[0]; + + for block in blocks { + let block = *block; + let mut page = root_page; + let mut depth = root_depth as u32; + + let mut offset = 0; + while depth > 0 { + self.read_block(page, &mut block_data)?; + + let index = (block - offset) / Self::pool_size().pow(depth as u32) as u32; + offset = Self::table_cell_offset(depth - 1, index as u32, offset) * Self::pool_size() as u32; + + page = u32::unpack(&block_data, &mut (4 * index as usize)).unwrap(); + + depth -= 1; + } + + self.read_block(page, &mut block_data)?; + let bit_index = block - offset; + let byte = bit_index / 8; + let bit = bit_index % 8; + + block_data[byte as usize] &= !(1 << bit); + self.write_block(page, &block_data)?; + } + + Ok(()) + } + fn acquire_object(&mut self, data_id:u32, length:usize) -> Result + // Allocate the next available object record. + // { //println!("acquire_object()"); + let mut block_data = [0u8; Z]; + let mut b8 = [0u8; 1]; let mut b32 = [0u8; 4]; @@ -439,14 +556,14 @@ impl BlockFile { //println!("basis: {}", basis); - let mut block = self.read_block(block_id)?; + self.read_block(block_id, &mut block_data)?; let mut write_block = false; let cell_index = (object_id - basis) / Self::table_offset(depth) as usize; //println!(" - ci {}", cell_index); let cell_start = cell_index * 4; - let cell_data = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default(); + let cell_data = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default(); // Allocate new page if pointer is zero. let child_id = if cell_data == 0 { @@ -454,10 +571,10 @@ impl BlockFile { // Write new reference to table. let pack_block = allocation[0].pack(); - block[cell_start] = pack_block[0]; - block[cell_start + 1] = pack_block[1]; - block[cell_start + 2] = pack_block[2]; - block[cell_start + 3] = pack_block[3]; + block_data[cell_start] = pack_block[0]; + block_data[cell_start + 1] = pack_block[1]; + block_data[cell_start + 2] = pack_block[2]; + block_data[cell_start + 3] = pack_block[3]; write_block = true; @@ -483,11 +600,11 @@ impl BlockFile { allocation[0] } else { - u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default() + u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default() }; if write_block { - self.write_block(block_id, &block)?; + self.write_block(block_id, &block_data)?; } // Update frame of reference to child table. @@ -499,26 +616,26 @@ impl BlockFile { //println!("end basis: {}", basis); // Update block and header with object information. - let mut block = self.read_block(block_id)?; + self.read_block(block_id, &mut block_data)?; let cell_index = (object_id - basis) / Self::table_offset(depth) as usize; let cell_start = cell_index * 8; - let next_pointer = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default(); + let next_pointer = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default(); //println!(" - next ptr: {}", next_pointer); // Update cell with data location and length. let pack_location = data_id.pack(); - block[cell_start] = pack_location[0]; - block[cell_start + 1] = pack_location[1]; - block[cell_start + 2] = pack_location[2]; - block[cell_start + 3] = pack_location[3]; + block_data[cell_start] = pack_location[0]; + block_data[cell_start + 1] = pack_location[1]; + block_data[cell_start + 2] = pack_location[2]; + block_data[cell_start + 3] = pack_location[3]; let pack_length = (length as u32).pack(); - block[cell_start + 4] = pack_length[0]; - block[cell_start + 5] = pack_length[1]; - block[cell_start + 6] = pack_length[2]; - block[cell_start + 7] = pack_length[3]; + block_data[cell_start + 4] = pack_length[0]; + block_data[cell_start + 5] = pack_length[1]; + block_data[cell_start + 6] = pack_length[2]; + block_data[cell_start + 7] = pack_length[3]; // Update header with new pointer. let pack_pointer = next_pointer.pack(); @@ -526,13 +643,17 @@ impl BlockFile { self.file.write(&pack_pointer)?; - self.write_block(block_id, &block)?; + self.write_block(block_id, &block_data)?; Ok(object_id) } fn get_object(&self, id:usize) -> Result<(u32, usize), std::io::Error> + // Find initial block and data size of an object. + // { + let mut block_data = [0u8; Z]; + //println!("get_object()"); let mut file = self.file.try_clone()?; @@ -556,15 +677,15 @@ impl BlockFile { ** Select child tables containing object_id until depth is 0. */ - let block = self.read_block(block_id)?; + self.read_block(block_id, &mut block_data)?; let cell_index = (id - basis) / Self::table_offset(depth) as usize; let cell_start = cell_index * 4; - let cell_data = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default(); + let cell_data = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default(); let child_id = if cell_data != 0 { - u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default() + u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default() } else { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "object id not valid")); }; @@ -576,28 +697,44 @@ impl BlockFile { } // Get object pointer and length from cell. - let block = self.read_block(block_id)?; + self.read_block(block_id, &mut block_data)?; let cell_index = (id - basis) / Self::table_offset(depth) as usize; let mut cell_start = cell_index * 8; - let pointer = u32::unpack(&block, &mut cell_start).unwrap_or_default(); - let length = u32::unpack(&block, &mut cell_start).unwrap_or_default(); + let pointer = u32::unpack(&block_data, &mut cell_start).unwrap_or_default(); + let length = u32::unpack(&block_data, &mut cell_start).unwrap_or_default(); Ok((pointer, length as usize)) } - fn read_block(&self, block_id:u32) -> Result, std::io::Error> + fn read_block(&self, block_id:u32, data:&mut [u8;Z]) -> Result<(),std::io::Error> + // Read a block from file. + // { let mut file = self.file.try_clone()?; - let mut data = vec![0u8; Z]; file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize)) as u64))?; + file.read_exact(data)?; + + Ok(()) + } + + fn read_block_pointer(&self, block_id:u32) -> Result + // Read the pointer element of a data block. + // + { + let mut file = self.file.try_clone()?; + + let mut data = [0u8; 4]; + file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize) + Self::data_size()) as u64))?; file.read(&mut data)?; - Ok(data) + Ok(u32::unpack(&data, &mut 0).unwrap()) } fn write_block(&mut self, block_id:u32, data:&[u8]) -> Result<(), std::io::Error> + // Write a block to file. + // { self.file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize)) as u64))?; self.file.write(&data[0..Z])?; @@ -606,6 +743,8 @@ impl BlockFile { } fn end_block(&self) -> Result + // Id of the last block in the file. + // { let mut file = self.file.try_clone()?; let index = file.seek(SeekFrom::End(0))? as usize; @@ -622,17 +761,30 @@ impl BlockFile { (Self::table_size() as u32).pow(depth) } + fn block_count_from_size(size:usize) -> usize + // Number of blocks required to hold a given number of bytes. + // + { + (size / (Z - 4)) + ((size % (Z - 4)) != 0) as usize + } + const fn data_size() -> usize + // Number of bytes of data per block. + // { Z - 4 } const fn table_size() -> usize + // Number of elements per table block. + // { Z / 8 } const fn pool_size() -> usize + // Number of blocks represented per pool block. + // { Z * 8 }