Implement methods for block file.

src/bin/test.rs

@@ -1,24 +1,20 @@
 //use storage::BlockFile;
-use storage::TrieFile;
+use storage::*;
 
 fn main()
 {
     std::fs::create_dir_all("data").ok();
 
-    /*if let Ok(mut bf) = BlockFile::<128>::open("data/cache_data.bin") {
+    if let Ok(mut bf) = BlockFile::<16>::open("data/cache_data.bin") {
-
+        if let Ok(id) = bf.insert("This is a test of the block file system.".as_bytes()) {
-        for i in 0..760 {
+            let data = String::from_utf8(bf.get(id).unwrap()).unwrap();
-            if let Ok(id) = bf.insert(format!("Hello, world! {}. This is long text to increase the block size to sufficient length to roll over into a second block when using smaller block sizes.", i).as_bytes()) {
+            println!("id {} = '{}'", id, data);
-
-                let data = String::from_utf8(bf.get(id).unwrap()).unwrap();
-                println!("id {} = '{}'", id, data);
-            }
         }
     } else {
         println!("Failed to open.");
-    }*/
+    }
 
-    if let Ok(mut tf) = TrieFile::<8>::open("data/cache_index.bin") {
+    /*if let Ok(mut tf) = TrieFile::<8>::open("data/cache_index.bin") {
 
         for s in [
             "Hello",
@@ -40,5 +36,5 @@ fn main()
 
     } else {
         println!("Failed to open index.");
-    }
+    }*/
 }

src/blockfile/mod.rs

@@ -70,6 +70,8 @@ impl<const Z:usize> BlockFile<Z> {
     }
 
     fn init(mut self) -> Result<Self, std::io::Error>
+    // Write initial headers to the file.
+    //
     {
         /*
         ** Header size and first two blocks are initialized for
@@ -98,10 +100,12 @@ impl<const Z:usize> BlockFile<Z> {
     }
 
     pub fn insert(&mut self, data:&[u8]) -> Result<usize, std::io::Error>
+    // Acquire a new object id and write its data to new allocations.
+    //
     {
 
         // Allocate storage blocks
-        let block_count = (data.len() / (Z - 4)) + ((data.len() % (Z - 4)) != 0) as usize;
+        let block_count = Self::block_count_from_size(data.len());
         //println!("block_count {}", block_count);
 
         let blocks = self.allocate(block_count.max(1))?;
@@ -112,10 +116,10 @@ impl<const Z:usize> BlockFile<Z> {
 
         //println!("obj_id {}", id);
 
-        let mut block_data = vec![0u8; Z];
-
         // Write data to storage blocks
         let mut data_index = 0;
+        let mut block_data = vec![0u8; Z];
+
         for block_index in 0..block_count {
             block_data.fill(0);
 
@@ -140,26 +144,83 @@ impl<const Z:usize> BlockFile<Z> {
         Ok(id)
     }
 
-    /*pub fn update(&mut self, _id:usize, _data:&[u8]) -> Result<(), std::io::Error>
+    pub fn update(&mut self, id:usize, data:&[u8]) -> Result<(), std::io::Error>
+    // Write new data to an object expanding its existing allocation.
+    //
     {
-        
+        // Get first block and data size
-    }*/
+        let (mut block_id, _) = self.get_object(id)?;
 
-    /*pub fn remove(&mut self, _id:usize) -> Result<(), std::io::Error>
+        let required_blocks = Self::block_count_from_size(data.len());
-    {
+        let mut blocks = Vec::new();
+
+        // Count blocks
+        while block_id != 0 {
+            let next = self.read_block_pointer(block_id)?;
+            blocks.push(block_id);
+            block_id = next;
+        }
+
+        if blocks.len() > required_blocks {
+            // Free excess blocks
+            self.release(&blocks[required_blocks..])?;
+            blocks.resize(required_blocks, 0);
+
+        } else if blocks.len() < required_blocks {
+            // Allocate additional blocks
+            let allocated_blocks = self.allocate(required_blocks - blocks.len())?;
+            blocks.extend_from_slice(&allocated_blocks);
+        }
+
+        // Write data to storage blocks
+        let mut data_index = 0;
+        let mut block_data = vec![0u8; Z];
         
-    }*/
+        for block_index in 0..blocks.len() {
+            block_data.fill(0);
+
+            // Copy data slice to buffer
+            for b in 0..Self::data_size().min(data.len() - (block_index * Self::data_size())) {
+                block_data[b] = data[data_index + b];
+            }
+            data_index += Self::data_size();
+
+            // Write pointer to next block to end of buffer
+            if block_index < blocks.len() - 1 {
+                let pack_next = (blocks[block_index + 1] as u32).pack();
+                block_data[Z - 4] = pack_next[0];
+                block_data[Z - 3] = pack_next[1];
+                block_data[Z - 2] = pack_next[2];
+                block_data[Z - 1] = pack_next[3];
+            }
+
+            self.write_block(blocks[block_index], &block_data)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn remove(&mut self, _id:usize) -> Result<(), std::io::Error>
+    // Remove the object with the specified identifier and release its allocation.
+    //
+    {
+        Ok(())
+    }
 
     pub fn get(&self, id:usize) -> Result<Vec<u8>, std::io::Error>
+    // Return the object with the specified identifier.
+    //
     {
+        let mut block_data = [0u8; Z];
+
         // Get first block and data size
         let (mut block_id, size) = self.get_object(id)?;
         let mut data = Vec::new();
 
         // Read blocks until size is full
         while block_id != 0 {
-            let block = self.read_block(block_id)?;
+            self.read_block(block_id, &mut block_data)?;
-            let next_block = u32::unpack(&block, &mut (Z - 4)).unwrap_or_default();
+            let next_block = u32::unpack(&block_data, &mut (Z - 4)).unwrap_or_default();
             
             //println!("size {} len {}", size, data.len());
 
@@ -169,7 +230,7 @@ impl<const Z:usize> BlockFile<Z> {
                 size - data.len()
             };
 
-            data.extend_from_slice(&block[0..data_length]);
+            data.extend_from_slice(&block_data[0..data_length]);
 
             block_id = next_block;
         }
@@ -178,6 +239,8 @@ impl<const Z:usize> BlockFile<Z> {
     }
 
     fn allocate(&mut self, count:usize) -> Result<Vec<u32>, std::io::Error>
+    // Mark as allocated and return the next available N blocks.
+    //
     {
         let mut b8 = [0u8; 1];
         let mut b32 = [0u8; 4];
@@ -213,10 +276,14 @@ impl<const Z:usize> BlockFile<Z> {
         count:usize,
         blocks:&mut Vec<u32>,
     ) -> Result<Operation, std::io::Error>
+    // Search allocation page tables for next available child page.
+    // Mark blocks allocated until requested number is acquired.
+    //
     {
         //println!("allocate_traverse()");
 
-        let mut block = self.read_block(block_id)?;
+        let mut block_data = [0u8; Z];
+        self.read_block(block_id, &mut block_data)?;
         let mut write_block = false;
 
         let mut operation = Operation::None;
@@ -237,7 +304,7 @@ impl<const Z:usize> BlockFile<Z> {
                 cell_index = i;
 
                 let cell_byte_index = byte_index;
-                let cell_data = u32::unpack(&block, &mut byte_index).expect("failed to unpack during alloc");
+                let cell_data = u32::unpack(&block_data, &mut byte_index).expect("failed to unpack during alloc");
 
                 //println!(" - cell {}", cell_data);
 
@@ -272,10 +339,10 @@ impl<const Z:usize> BlockFile<Z> {
 
                     // Update cell with child table reference.
                     let pack_child = next_block.pack();
-                    block[cell_byte_index] = pack_child[0];
+                    block_data[cell_byte_index] = pack_child[0];
-                    block[cell_byte_index + 1] = pack_child[1];
+                    block_data[cell_byte_index + 1] = pack_child[1];
-                    block[cell_byte_index + 2] = pack_child[2];
+                    block_data[cell_byte_index + 2] = pack_child[2];
-                    block[cell_byte_index + 3] = pack_child[3];
+                    block_data[cell_byte_index + 3] = pack_child[3];
                     write_block = true;
 
                     // If root table allocated last page, generate new root table at greater depth.
@@ -300,10 +367,10 @@ impl<const Z:usize> BlockFile<Z> {
                         self.write_block(parent_blocks[0], &table_data)?;
         
                         // Update current table before restarting recursion.
-                        self.write_block(block_id, &block)?;
+                        self.write_block(block_id, &block_data)?;
 
                         // Restart recursion with new root.
-                        self.allocate_traverse(parent_blocks[0], depth + 1, basis, true, count, blocks)?;
+                        self.allocate_traverse(parent_blocks[0], depth + 1, 0, true, count, blocks)?;
                         return Ok(Operation::None);
                     }
                 }
@@ -318,7 +385,7 @@ impl<const Z:usize> BlockFile<Z> {
                         Operation::SetOccupied => {
                             write_block = true;
 
-                            block[cell_byte_index + 3] |= 0x80;
+                            block_data[cell_byte_index + 3] |= 0x80;
 
                             // If last cell is marked occupied, this table is also occupied.
                             if cell_index == Self::table_size() {
@@ -342,9 +409,9 @@ impl<const Z:usize> BlockFile<Z> {
 
             // Find first unset bit in block.
             let mut byte_index = 0;
-            while byte_index < block.len() && blocks.len() < count {
+            while byte_index < block_data.len() && blocks.len() < count {
-                if block[byte_index] != 0xFF {
+                if block_data[byte_index] != 0xFF {
-                    let bit = block[byte_index].trailing_ones();
+                    let bit = block_data[byte_index].trailing_ones();
                     let id = basis + (byte_index * 8) as u32 + bit;
 
                     //println!(" - cell {} value {:02x} bit {} alloc_id {}", byte_index, block[byte_index], bit, id);
@@ -357,7 +424,7 @@ impl<const Z:usize> BlockFile<Z> {
                     //println!(" - basis {} byte_index {} byte {:02x} bit {} alloc {}", basis, byte_index, block[byte_index], bit, id);
 
                     // Mark block as occupied.
-                    block[byte_index] |= 1 << bit;
+                    block_data[byte_index] |= 1 << bit;
                     write_block = true;
                 } else {
                     byte_index += 1;
@@ -365,23 +432,73 @@ impl<const Z:usize> BlockFile<Z> {
             }
 
             // If table is fully allocated, signal parent to mark table as not vacant.
-            if byte_index == block.len() && block[block.len() - 1] == 0xFF {
+            if byte_index == Z && block_data[Z - 1] == 0xFF {
                 operation = Operation::SetOccupied;
                 //println!("OCCUPIED!");
             }
         }
 
         if write_block {
-            self.write_block(block_id, &block)?;
+            self.write_block(block_id, &block_data)?;
         }
 
         Ok(operation)
     }
 
+    fn release(&mut self, blocks:&[u32]) -> Result<(),std::io::Error>
+    // Mark a set of blocks as unallocated.
+    //
+    {
+        let mut b8 = [0u8; 1];
+        let mut b32 = [0u8; 4];
+
+        let mut block_data = [0u8; Z];
+
+        // Read allocation table root block and depth from file.
+        self.file.seek(SeekFrom::Start(0))?;
+        self.file.read_exact(&mut b8)?;
+        self.file.read_exact(&mut b32)?;
+
+        let root_page = u32::unpack(&b32, &mut 0).unwrap_or_default();
+        let root_depth = b8[0];
+
+        for block in blocks {
+            let block = *block;
+            let mut page = root_page;
+            let mut depth = root_depth as u32;
+
+            let mut offset = 0;
+            while depth > 0 {
+                self.read_block(page, &mut block_data)?;
+
+                let index = (block - offset) / Self::pool_size().pow(depth as u32) as u32;
+                offset = Self::table_cell_offset(depth - 1, index as u32, offset) * Self::pool_size() as u32;
+
+                page = u32::unpack(&block_data, &mut (4 * index as usize)).unwrap();
+
+                depth -= 1;
+            }
+
+            self.read_block(page, &mut block_data)?;
+            let bit_index = block - offset;
+            let byte = bit_index / 8;
+            let bit = bit_index % 8;
+
+            block_data[byte as usize] &= !(1 << bit);
+            self.write_block(page, &block_data)?;
+        }
+
+        Ok(())
+    }
+
     fn acquire_object(&mut self, data_id:u32, length:usize) -> Result<usize, std::io::Error>
+    // Allocate the next available object record.
+    //
     {
         //println!("acquire_object()");
 
+        let mut block_data = [0u8; Z];
+
         let mut b8 = [0u8; 1];
         let mut b32 = [0u8; 4];
 
@@ -439,14 +556,14 @@ impl<const Z:usize> BlockFile<Z> {
 
             //println!("basis: {}", basis);
 
-            let mut block = self.read_block(block_id)?;
+            self.read_block(block_id, &mut block_data)?;
             let mut write_block = false;
 
             let cell_index = (object_id - basis) / Self::table_offset(depth) as usize;
             //println!(" - ci {}", cell_index);
 
             let cell_start = cell_index * 4;
-            let cell_data = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default();
+            let cell_data = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default();
 
             // Allocate new page if pointer is zero.
             let child_id = if cell_data == 0 {
@@ -454,10 +571,10 @@ impl<const Z:usize> BlockFile<Z> {
 
                 // Write new reference to table.
                 let pack_block = allocation[0].pack();
-                block[cell_start] = pack_block[0];
+                block_data[cell_start] = pack_block[0];
-                block[cell_start + 1] = pack_block[1];
+                block_data[cell_start + 1] = pack_block[1];
-                block[cell_start + 2] = pack_block[2];
+                block_data[cell_start + 2] = pack_block[2];
-                block[cell_start + 3] = pack_block[3];
+                block_data[cell_start + 3] = pack_block[3];
 
                 write_block = true;
 
@@ -483,11 +600,11 @@ impl<const Z:usize> BlockFile<Z> {
 
                 allocation[0]
             } else {
-                u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default()
+                u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default()
             };
 
             if write_block {
-                self.write_block(block_id, &block)?;
+                self.write_block(block_id, &block_data)?;
             }
 
             // Update frame of reference to child table.
@@ -499,26 +616,26 @@ impl<const Z:usize> BlockFile<Z> {
         //println!("end basis: {}", basis);
 
         // Update block and header with object information.
-        let mut block = self.read_block(block_id)?;
+        self.read_block(block_id, &mut block_data)?;
         
         let cell_index = (object_id - basis) / Self::table_offset(depth) as usize;
         let cell_start = cell_index * 8;
-        let next_pointer = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default();
+        let next_pointer = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default();
 
         //println!(" - next ptr: {}", next_pointer);
 
         // Update cell with data location and length.
         let pack_location = data_id.pack();
-        block[cell_start] = pack_location[0];
+        block_data[cell_start] = pack_location[0];
-        block[cell_start + 1] = pack_location[1];
+        block_data[cell_start + 1] = pack_location[1];
-        block[cell_start + 2] = pack_location[2];
+        block_data[cell_start + 2] = pack_location[2];
-        block[cell_start + 3] = pack_location[3];
+        block_data[cell_start + 3] = pack_location[3];
 
         let pack_length = (length as u32).pack();
-        block[cell_start + 4] = pack_length[0];
+        block_data[cell_start + 4] = pack_length[0];
-        block[cell_start + 5] = pack_length[1];
+        block_data[cell_start + 5] = pack_length[1];
-        block[cell_start + 6] = pack_length[2];
+        block_data[cell_start + 6] = pack_length[2];
-        block[cell_start + 7] = pack_length[3];
+        block_data[cell_start + 7] = pack_length[3];
 
         // Update header with new pointer.
         let pack_pointer = next_pointer.pack();
@@ -526,13 +643,17 @@ impl<const Z:usize> BlockFile<Z> {
         self.file.write(&pack_pointer)?;
 
 
-        self.write_block(block_id, &block)?;
+        self.write_block(block_id, &block_data)?;
 
         Ok(object_id)
     }
 
     fn get_object(&self, id:usize) -> Result<(u32, usize), std::io::Error>
+    // Find initial block and data size of an object.
+    //
     {
+        let mut block_data = [0u8; Z];
+
         //println!("get_object()");
 
         let mut file = self.file.try_clone()?;
@@ -556,15 +677,15 @@ impl<const Z:usize> BlockFile<Z> {
             ** Select child tables containing object_id until depth is 0.
             */
 
-            let block = self.read_block(block_id)?;
+            self.read_block(block_id, &mut block_data)?;
 
             let cell_index = (id - basis) / Self::table_offset(depth) as usize;
 
             let cell_start = cell_index * 4;
-            let cell_data = u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default();
+            let cell_data = u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default();
 
             let child_id = if cell_data != 0 {
-                u32::unpack(&block, &mut cell_start.clone()).unwrap_or_default()
+                u32::unpack(&block_data, &mut cell_start.clone()).unwrap_or_default()
             } else {
                 return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "object id not valid"));
             };
@@ -576,28 +697,44 @@ impl<const Z:usize> BlockFile<Z> {
         }
 
         // Get object pointer and length from cell.
-        let block = self.read_block(block_id)?;
+        self.read_block(block_id, &mut block_data)?;
         
         let cell_index = (id - basis) / Self::table_offset(depth) as usize;
         let mut cell_start = cell_index * 8;
-        let pointer = u32::unpack(&block, &mut cell_start).unwrap_or_default();
+        let pointer = u32::unpack(&block_data, &mut cell_start).unwrap_or_default();
-        let length = u32::unpack(&block, &mut cell_start).unwrap_or_default();
+        let length = u32::unpack(&block_data, &mut cell_start).unwrap_or_default();
 
         Ok((pointer, length as usize))
     }
 
-    fn read_block(&self, block_id:u32) -> Result<Vec<u8>, std::io::Error>
+    fn read_block(&self, block_id:u32, data:&mut [u8;Z]) -> Result<(),std::io::Error>
+    // Read a block from file.
+    //
     {
         let mut file = self.file.try_clone()?;
 
-        let mut data = vec![0u8; Z];
         file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize)) as u64))?;
+        file.read_exact(data)?;
+
+        Ok(())
+    }
+
+    fn read_block_pointer(&self, block_id:u32) -> Result<u32,std::io::Error>
+    // Read the pointer element of a data block.
+    //
+    {
+        let mut file = self.file.try_clone()?;
+
+        let mut data = [0u8; 4];
+        file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize) + Self::data_size()) as u64))?;
         file.read(&mut data)?;
 
-        Ok(data)
+        Ok(u32::unpack(&data, &mut 0).unwrap())
     }
 
     fn write_block(&mut self, block_id:u32, data:&[u8]) -> Result<(), std::io::Error>
+    // Write a block to file.
+    //
     {
         self.file.seek(SeekFrom::Start((HEADER_SIZE + (Z * block_id as usize)) as u64))?;
         self.file.write(&data[0..Z])?;
@@ -606,6 +743,8 @@ impl<const Z:usize> BlockFile<Z> {
     }
 
     fn end_block(&self) -> Result<u32, std::io::Error>
+    // Id of the last block in the file.
+    //
     {
         let mut file = self.file.try_clone()?;
         let index = file.seek(SeekFrom::End(0))? as usize;
@@ -622,17 +761,30 @@ impl<const Z:usize> BlockFile<Z> {
         (Self::table_size() as u32).pow(depth)
     }
 
+    fn block_count_from_size(size:usize) -> usize
+    // Number of blocks required to hold a given number of bytes.
+    //
+    {
+        (size / (Z - 4)) + ((size % (Z - 4)) != 0) as usize
+    }
+
     const fn data_size() -> usize
+    // Number of bytes of data per block.
+    //
     {
         Z - 4
     }
 
     const fn table_size() -> usize
+    // Number of elements per table block.
+    //
     {
         Z / 8
     }
 
     const fn pool_size() -> usize
+    // Number of blocks represented per pool block.
+    //
     {
         Z * 8
     }