Add Sparse, update README, remove risky methods, fix bugs.

README.md

@@ -234,19 +234,6 @@ match Integer::from(list.at(0)) {
 ```
 ---
 
-`with(value) -> Self`
-
-```
-let b = Boolean::with(true);
-```
----
-
-`detatch()`
-
-Prevents an allocated object from being dropped when the interface goes out of scope.
-
----
-
 `*Dereference -> Reference`
 
 ```
@@ -372,7 +359,7 @@ Stores a constant-magnitude number with whole and decimal components.
 ## Significant
 Stores a fixed-precision, variable-magnitude number.
 
-> Encode not implemented.
+> Encode/decode not implemented.
 
 `get() -> f64`
 
@@ -486,7 +473,7 @@ Reallocates the sequeunce to have capacity not less than the specified size.
 
 
 ## Array
-Constant-sized, ordered collection of items.
+Constant-sized, indexed collection of items.
 
 `new(length:usize, type_id:usize) -> Array`
 
@@ -520,7 +507,7 @@ Returns the type identifier of the contents.
 
 
 ## List
-Variable-sized, ordered collection of items.
+Variable-sized, indexed collection of items.
 
 `new(type_id:usize) -> List`
 
@@ -577,9 +564,63 @@ Removes all elements from the list.
 ---
 
 
+## Sparse
+List of discontinuous indicies.
+
+`length() -> usize`
+
+Returns the number of elements in the collection.
+
+---
+
+`at(index:usize) -> Reference`
+
+Returns a reference to the item at the specified index.
+
+---
+
+`has(key:usize) -> bool`
+
+Returns whether the specified key is assigned.
+
+---
+
+`get(key:usize) -> Reference`
+
+Returns a reference to the item at the specified key.
+
+---
+
+`set(key:usize, source:Reference)`
+
+Adds or updates an element at the specified key.
+
+---
+
+`unset(key:usize)`
+
+Removes an element from the collection.
+
+---
+
+`clear()`
+
+Removes all elements from the collection.
+
+---
+
+`indexof(index:usize) -> usize`
+
+Returns the key of the element at the specified index.
+
+---
+
+
 ## Schema
 Definition of an abstract structure composed of named items.
 
+> Encode/decode not implemented.
+
 `with(members:Vec<(&str, usize)>) -> Schema`
 
 Produces a schema with the provided member assignments.

src/encoding/mod.rs

@@ -1,8 +1,9 @@
 use crate::{
+    runtime,
     runtime::{
         Reference,
         kind_hasinner, type_hasinner, type_inner, type_outer,
-        type_key, type_innerkey,
+        type_key, type_innerkey, SparseHeader,
     },
     tag,
     Type,
@@ -84,6 +85,29 @@ pub fn encode_data(addr:Reference) -> Vec<u8>
                 }
             }
         }
+        tag::SPARSE => {
+            let data = crate::Sparse::from(addr).unwrap();
+
+            let header_length = unsafe {runtime::sparse_header_length(addr)};
+
+            result.append(&mut util::pack_natural(header_length as u64));
+
+            for i in 0..header_length {
+                let header = unsafe {runtime::sparse_header_data(addr, i)};
+                result.append(&mut util::pack_natural(header.start as u64));
+                result.append(&mut util::pack_natural(header.length as u64));
+                result.append(&mut util::pack_natural(header.index as u64));
+            }
+
+            for i in 0..data.length() {
+                if unsafe {type_innerkey(addr.class)} == tag::VARYING {
+                    result.append(&mut encode(data.at(i)));
+                }
+                else {
+                    result.append(&mut encode_data(data.at(i)));
+                }
+            }
+        }
         tag::RECORD => {
             let data = Record::from(addr).unwrap();
             for i in 0..data.length() {
@@ -171,6 +195,76 @@ pub fn decode_data(data:&Vec<u8>, type_id:usize, index:&mut usize) -> Result<Typ
             }
             return Ok(Type::Sequence(Sequence::with_raw(bytes)));
         }
+        tag::ARRAY => {
+            let length = unsafe {type_innerkey(type_id)};
+            let inner = unsafe {type_inner(type_inner(type_id))};
+
+            let mut result = Array::new(length, inner);
+            for i in 0..length {
+                match if unsafe {type_key(inner)} == tag::VARYING {
+                    decode(data, index)
+                } else {
+                    decode_data(data, inner, index)
+                } {
+                    Ok(data) => { result.set(i, data.get()); }
+                    Err(_) => { return Err(()); }
+                }
+            }
+
+            return Ok(Type::Array(result));
+        }
+        tag::LIST => {
+            let inner = unsafe {type_inner(type_id)};
+            let length = util::unpack_natural(data, index) as usize;
+
+            let mut result = List::new(inner);
+            result.reserve(length);
+
+            for _ in 0..length {
+                match if unsafe {type_key(inner)} == tag::VARYING {
+                    decode(data, index)
+                } else {
+                    decode_data(data, inner, index)
+                } {
+                    Ok(data) => { result.append(data.get()); }
+                    Err(_) => { return Err(()); }
+                }
+            }
+
+            return Ok(Type::List(result));
+        }
+        tag::SPARSE => {
+            let inner = unsafe {type_inner(type_id)};
+            let mut result = crate::Sparse::new(inner);
+
+            let header_length = util::unpack_natural(data, index) as usize;
+            let mut headers = Vec::<SparseHeader>::with_capacity(header_length);
+            
+            for _ in 0..header_length {
+                headers.push(runtime::SparseHeader {
+                    start:util::unpack_natural(data, index) as usize,
+                    length:util::unpack_natural(data, index) as usize,
+                    index:util::unpack_natural(data, index) as usize,
+                });
+            }
+
+            for header in headers {
+                for i in 0..header.length {
+                    match if unsafe {type_key(inner)} == tag::VARYING {
+                        decode(data, index)
+                    } else {
+                        decode_data(data, inner, index)
+                    } {
+                        Ok(data) => {
+                            result.set(header.start + i, data.get());
+                        }
+                        Err(_) => { return Err(()); }
+                    }
+                }
+            }
+
+            return Ok(Type::Sparse(result));
+        }
         tag::RECORD => {
             return match Record::new(unsafe {type_innerkey(type_id)}) {
                 Ok(mut value) => {

src/interface/array.rs

@@ -53,17 +53,6 @@ impl Array {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn length(&self) -> usize
     {
         unsafe {array_length(self.addr)}

src/interface/block.rs

@@ -38,17 +38,6 @@ impl Block {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn size(&self) -> usize
     {
         unsafe {block_length(self.addr)}

src/interface/boolean.rs

@@ -37,17 +37,6 @@ impl Boolean {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn set(&mut self, value:bool)
     {
         unsafe { bool_set(self.addr, value) };

src/interface/builder.rs

@@ -28,6 +28,7 @@ pub fn array(size:usize, type_id:usize) -> usize {
 }
 
 pub fn list(type_id:usize) -> usize { unsafe { runtime::type_outer(type_id, tag::LIST) } }
+pub fn sparse(type_id:usize) -> usize { unsafe { runtime::type_outer(type_id, tag::SPARSE) } }
 
 pub fn record(schema_id:usize) -> usize {
     unsafe {

src/interface/integer.rs

@@ -37,17 +37,6 @@ impl Integer {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn set(&mut self, value:i64)
     {
         unsafe { integer_set(self.addr, value) };

src/interface/list.rs

@@ -44,17 +44,6 @@ impl List {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn capacity(&self) -> usize
     {
         unsafe {list_capacity(self.addr)}

src/interface/mod.rs

@@ -8,11 +8,12 @@ mod varying; pub use varying::Varying;
 mod boolean; pub use boolean::Boolean;
 mod natural; pub use natural::Natural;
 mod integer; pub use integer::Integer;
-mod signfiicant; pub use signfiicant::Significant;
+mod signficant; pub use signficant::Significant;
 mod block; pub use block::Block;
 mod sequence; pub use sequence::Sequence;
 mod array; pub use array::Array;
 mod list; pub use list::List;
+mod sparse; pub use sparse::Sparse;
 mod schema; pub use schema::Schema;
 mod record; pub use record::Record;
 
@@ -26,6 +27,7 @@ pub enum Type {
     Sequence(Sequence),
     Array(Array),
     List(List),
+    Sparse(Sparse),
     Record(Record),
     Schema(Schema),
 }
@@ -42,6 +44,7 @@ impl Type {
             tag::SEQUENCE => Type::Sequence(Sequence::from(addr).unwrap()),
             tag::ARRAY => Type::Array(Array::from(addr).unwrap()),
             tag::LIST => Type::List(List::from(addr).unwrap()),
+            tag::SPARSE => Type::Sparse(Sparse::from(addr).unwrap()),
             tag::RECORD => Type::Record(Record::from(addr).unwrap()),
             _ => Type::Null,
         }
@@ -58,6 +61,7 @@ impl Type {
             Type::Sequence(obj) => **obj,
             Type::Array(obj) => **obj,
             Type::List(obj) => **obj,
+            Type::Sparse(obj) => **obj,
             Type::Record(obj) => **obj,
             _ => Reference::null(),
         }

src/interface/natural.rs

@@ -37,17 +37,6 @@ impl Natural {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn set(&mut self, value:u64)
     {
         unsafe { natural_set(self.addr, value) };

src/interface/record.rs

@@ -64,17 +64,6 @@ impl Record {
         }
     }
     
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn length(&self) -> usize
     {
         unsafe {record_length(self.addr)}
@@ -116,8 +105,7 @@ impl Record {
         let result = unsafe {record_indexof(self.addr, key_index)};
         if result != self.length() {
             Some(result)
-        }
+        } else {
-        else {
             None
         }
     }

src/interface/schema.rs

@@ -45,17 +45,6 @@ impl Schema {
         return obj;
     }
     
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-    
     pub fn length(&self) -> usize
     {
         unsafe {schema_length(self.addr)}

src/interface/sequence.rs

@@ -47,17 +47,6 @@ impl Sequence {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn capacity(&self) -> usize
     {
         unsafe {sequence_capacity(self.addr)}

src/interface/signficant.rs

@@ -37,17 +37,6 @@ impl Significant {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn set(&mut self, value:f64)
     {
         unsafe { significant_set(self.addr, value) };

src/interface/sparse.rs

@@ -0,0 +1,92 @@
+use crate::runtime::{
+    Reference,
+    acquire, release,
+    type_key,
+    sparse_length,
+    sparse_at, sparse_get,
+    sparse_clear,
+    sparse_set, sparse_unset,
+    sparse_indexof,
+};
+use crate::tag;
+use super::sparse;
+
+pub struct Sparse {
+    managed:bool,
+    addr:Reference,
+}
+impl Sparse {
+    pub fn new(class:usize) -> Self
+    {
+        Self {
+            managed:true,
+            addr:unsafe {acquire(sparse(class))},
+        }
+    }
+
+    pub fn from(addr:Reference) -> Result<Self,()>
+    {
+        return if(unsafe {type_key(addr.class)} == tag::SPARSE) {
+            Ok(Self { managed:false, addr:addr })
+        }
+        else {
+            Err(())
+        }
+    }
+
+    /* pub fn with(class:usize, data:Vec<Reference>) -> Self
+    {
+        let mut obj = Self::new(class);
+        for item in data {
+            obj.insert(obj.length(), item);
+        }
+        return obj;
+    } */
+
+    pub fn length(&self) -> usize
+    {
+        unsafe {sparse_length(self.addr)}
+    }
+
+    pub fn at(&self, index:usize) -> Reference
+    {
+        unsafe {sparse_at(self.addr, index)}
+    }
+
+    pub fn has(&self, key:usize) -> bool
+    {
+        unsafe {sparse_get(self.addr, key)}.address != 0
+    }
+
+    pub fn get(&self, key:usize) -> Reference
+    {
+        unsafe {sparse_get(self.addr, key)}
+    }
+
+    pub fn clear(&mut self)
+    {
+        unsafe{sparse_clear(self.addr)};
+    }
+
+    pub fn set(&mut self, key:usize, source:Reference)
+    {
+        unsafe{sparse_set(self.addr, key, source)};
+    }
+
+    pub fn unset(&mut self, key:usize)
+    {
+        unsafe{sparse_unset(self.addr, key)};
+    }
+
+    pub fn indexof(&self, index:usize) -> usize
+    {
+        unsafe{sparse_indexof(self.addr, index)}
+    }
+}
+impl std::ops::Deref for Sparse {
+    type Target = Reference;
+    fn deref(&self) -> &Self::Target { return &self.addr; }
+}
+impl Drop for Sparse {
+    fn drop(&mut self) { if self.managed { unsafe {release(self.addr)}; } }
+}

src/interface/varying.rs

@@ -38,17 +38,6 @@ impl Varying {
         return obj;
     }
 
-    pub fn detach(&mut self)
-    {
-        self.managed = false;
-    }
-
-    pub fn release(mut self)
-    {
-        self.detach();
-        unsafe { release(self.addr); }
-    }
-
     pub fn is_null(&self) -> bool
     {
         (unsafe {varying_get(self.addr)}).address == 0

src/lib.rs

@@ -7,24 +7,53 @@ mod interface; pub use interface::*;
 mod encoding; pub use encoding::*;
 
 pub fn test() {
-    let mut data = Varying::new();
+    use std::fs::File;
-    data.set(*Natural::with(79));
+    use std::io::prelude::*;
 
+    const MAGAZINE :usize     = 0x100;
+    const MAGAZINE_ROW :usize = 0x101;
+
+    // define schema "Magazine Row"
+    Schema::with(vec![
+        ("Content", natural()),
+        ("Quantity", natural()),
+    ]).bind(MAGAZINE_ROW);
+
+    // define schema "Magazine"
+    Schema::with(vec![
+        ("Capacity", natural()),
+        ("Quantity", natural()),
+        ("Content", list(record(MAGAZINE_ROW))),
+    ]).bind(MAGAZINE);
+
+    // create record "Magazine"
+    let data = Record::with(MAGAZINE, vec![
+        ("Capacity", *Natural::with(30)),
+        ("Quantity", *Natural::with(25)),
+        ("Content", *List::with(record(MAGAZINE_ROW), vec![
+            *Record::with(MAGAZINE_ROW, vec![
+                ("Content", *Natural::with(15)),
+                ("Quantity", *Natural::with(5)),
+            ]).unwrap(),
+            *Record::with(MAGAZINE_ROW, vec![
+                ("Content", *Natural::with(16)),
+                ("Quantity", *Natural::with(20)),
+            ]).unwrap(),
+        ])),
+    ]).unwrap();
+
+    // encode record
     let out = encode(*data);
+
+    // write encoding to file
+    let mut file = File::create("dat.szn").unwrap();
+    file.write_all(&out).ok();
+
+    // print hex series of encoding
     print!("[{}]: ", out.len());
     for byte in &out {
         print!("{:02x} ", *byte);
-    } println!("\n");
+    } println!("");
-
-    match decode(&out, &mut 0) {
-        Ok(ty) => match ty {
-            Type::Varying(data) => {
-                println!("Ok: {}", Natural::from(data.get()).unwrap().get());
-            }
-            _ => { println!("Other"); }
-        },
-        Err(_) => { println!("Failure"); }
-    }
 }
 
 #[cfg(test)]

src/runtime/lib.cc

@@ -55,6 +55,7 @@ extern "C" size_t type_size(size_t type_id)
             }
 
             case Type::Tag::List:
+            case Type::Tag::Sparse:
             case Type::Tag::Record:
             {
                 size_t innerkey = type_innerkey(type_id);
@@ -64,6 +65,7 @@ extern "C" size_t type_size(size_t type_id)
                 else {
                     switch(type) {
                         case Type::Tag::List: return sizeof(Type::List);
+                        case Type::Tag::Sparse: return sizeof(Type::Sparse);
                         case Type::Tag::Record: {
                             auto binding = DB_SCHEMA.get(type_innerkey(type_id));
                             if(binding != nullptr) {
@@ -88,23 +90,25 @@ size_t type_alignment(size_t type_id)
 
         switch(type) {
             case Type::Tag::Null:     return 0;
-            case Type::Tag::Varying:  return sizeof(size_t);
+
             case Type::Tag::Boolean:  return sizeof(Type::Boolean);
             case Type::Tag::Natural:  return sizeof(Type::Natural);
             case Type::Tag::Integer:  return sizeof(Type::Integer);
             case Type::Tag::Significant:  return sizeof(Type::Significant);
             case Type::Tag::Block:    return sizeof(uint8_t);
-            case Type::Tag::Sequence: return sizeof(size_t);
+            case Type::Tag::Array:    return type_alignment(type_inner(type_id));
-            case Type::Tag::Array:    return type_inner(type_id);
+
-            case Type::Tag::List:     return sizeof(size_t);
+            case Type::Tag::Varying:
+            case Type::Tag::Sequence:
+            case Type::Tag::List:
+            case Type::Tag::Sparse:
+            case Type::Tag::Schema:
+                return sizeof(size_t);
+
             case Type::Tag::Record: {
                 auto binding = DB_SCHEMA.get(type_innerkey(type_id));
-                if(binding != nullptr) {
+                return (binding != nullptr)? binding->alignment : 0;
-                    return binding->alignment;
-                }
             }
-            case Type::Tag::Schema: return sizeof(size_t);
-            default: return 0;
         }
     }
     return 0;
@@ -117,6 +121,7 @@ extern "C" size_t kind_hasinner(size_t kind)
             return 2;
         case Type::Tag::Block:
         case Type::Tag::List:
+        case Type::Tag::Sparse:
         case Type::Tag::Record:
             return 1;
     }
@@ -192,25 +197,19 @@ void drop(Reference addr)
     if(addr.address != nullptr) {
         switch(type_key(addr.type)) {
             case Type::Tag::Varying: {
-                auto& var = *reinterpret_cast<Reference*>(addr.address);
+                varying_clear(addr);
-                if(var.address != nullptr) {
-                    drop(var);
-                }
-                var.type = 0;
-                var.address = nullptr;
             } break;
 
             case Type::Tag::Sequence: {
-                auto& seq = *reinterpret_cast<Type::Sequence*>(addr.address);
+                sequence_clear(addr);
-                rawlist_clear(seq.data);
             } break;
 
             case Type::Tag::List: {
-                Type::List& list = *reinterpret_cast<Type::List*>(addr.address);
+                list_clear(addr);
-                for(size_t i = 0; i < list.data.length; ++i) {
+            } break;
-                    drop(list_cell(addr, i));
+
-                }
+            case Type::Tag::Sparse: {
-                rawlist_clear(list.data);
+                sparse_clear(addr);
             } break;
 
             case Type::Tag::Record: {
@@ -303,6 +302,27 @@ extern "C" bool copy(Reference dst, Reference src)
                     }
                 } break;
 
+                case Type::Tag::Sparse: {
+                    auto& src_list = *reinterpret_cast<Type::Sparse*>(source.address);
+                    auto& dst_list = *reinterpret_cast<Type::Sparse*>(destination.address);
+
+                    rawlist_reserve(dst_list.header, sizeof(Type::SparseHeader), src_list.header.capacity);
+                    dst_list.header.length = src_list.header.length;
+
+                    for(size_t i = 0; i < src_list.header.length; ++i) {
+                        Type::SparseHeader& src_header = *reinterpret_cast<Type::SparseHeader*>(rawlist_cell(src_list.header, sizeof(Type::SparseHeader), i));
+                        Type::SparseHeader& dst_header = *reinterpret_cast<Type::SparseHeader*>(rawlist_cell(dst_list.header, sizeof(Type::SparseHeader), i));
+                        dst_header = src_header;
+                    }
+
+                    rawlist_reserve(dst_list.data, type_size(type_inner(source.type)), src_list.data.capacity);
+                    dst_list.data.length = src_list.data.length;
+
+                    for(size_t i = 0; i < src_list.data.length; ++i) {
+                        copy(list_at(destination, i), list_at(source, i));
+                    }
+                } break;
+
                 case Type::Tag::Record: {
                     size_t schema_id = type_innerkey(source.type);
                     auto binding = DB_SCHEMA.get(schema_id);
@@ -439,9 +459,9 @@ extern "C" void varying_clear(Reference addr)
     if(var.address != nullptr) {
         drop(var);
         free(var.address);
-        var.type = 0;
-        var.address = nullptr;
     }
+    var.type = 0;
+    var.address = nullptr;
 }
 
 
@@ -669,33 +689,13 @@ extern "C" Reference list_at(Reference addr, size_t index)
     return result;
 }
 
-/*extern "C" Reference list_first(Reference addr)
-{
-    Reference result {0};
-    if(type_key(addr.type) == Type::Tag::List) {
-        auto& list = (*reinterpret_cast<Type::List*>(addr.address));
-        if(list.length > 0) {
-            result = list_at(addr, 0);
-        }
-    }
-    return result;
-}*/
-
-/*extern "C" Reference list_last(Reference addr)
-{
-    Reference result {0};
-    if(type_key(addr.type) == Type::Tag::List) {
-        auto& list = (*reinterpret_cast<Type::List*>(addr.address));
-        if(list.length > 0) {
-            result = list_at(addr, list.length - 1);
-        }
-    }
-    return result;
-}*/
-
 extern "C" void list_clear(Reference addr)
 {
-    drop(addr);
+    Type::List& list = *reinterpret_cast<Type::List*>(addr.address);
+    for(size_t i = 0; i < list.data.length; ++i) {
+        drop(list_cell(addr, i));
+    }
+    rawlist_clear(list.data);
 }
 
 extern "C" void list_insert(Reference addr, size_t index, Reference source)
@@ -706,13 +706,15 @@ extern "C" void list_insert(Reference addr, size_t index, Reference source)
 
     if(index > list.data.length) { index = list.data.length; }
 
-    void* cell = rawlist_insert(list.data, offset, index);
+    Reference cell {0};
+    cell.type = inner;
+    cell.address = rawlist_insert(list.data, offset, index);
 
     if(type_key(inner) == Type::Tag::Varying) {
-        varying_set(list_cell(addr, index), source);
+        varying_set(cell, source);
     }
     else {
-        copy(list_cell(addr, index), source);
+        copy(cell, source);
     }
 }
 
@@ -723,16 +725,15 @@ extern "C" void list_prepend(Reference addr, Reference source)
 
 extern "C" void list_append(Reference addr, Reference source)
 {
-    auto& list = (*reinterpret_cast<Type::List*>(addr.address));
+    auto& list = *reinterpret_cast<Type::List*>(addr.address);
     list_insert(addr, list.data.length, source);
 }
 
 extern "C" void list_update(Reference addr, size_t index, Reference source)
 {
-    auto& list = (*reinterpret_cast<Type::List*>(addr.address));
+    auto& list = *reinterpret_cast<Type::List*>(addr.address);
     if(index < list.data.length) {
         size_t inner = type_inner(addr.type);
-        size_t offset = type_size(inner);
 
         if(type_key(inner) == Type::Tag::Varying) {
             varying_set(list_cell(addr, index), source);
@@ -743,25 +744,9 @@ extern "C" void list_update(Reference addr, size_t index, Reference source)
     }
 }
 
-/*extern "C" void list_truncate(Reference addr, size_t maximum)
-{
-    if(type_key(addr.type) == Type::Tag::List) {
-        //auto& list = (*reinterpret_cast<Type::List*>(addr.address));
-        
-    }
-}*/
-
-/*extern "C" void list_shift(Reference addr)
-{
-    if(type_key(addr.type) == Type::Tag::List) {
-        //auto& list = (*reinterpret_cast<Type::List*>(addr.address));
-        
-    }
-}*/
-
 extern "C" void list_remove(Reference addr, size_t index)
 {
-    auto& list = (*reinterpret_cast<Type::List*>(addr.address));
+    auto& list = *reinterpret_cast<Type::List*>(addr.address);
     size_t inner = type_inner(addr.type);
     size_t offset = type_size(inner);
     if(index < list.data.length) {
@@ -772,20 +757,265 @@ extern "C" void list_remove(Reference addr, size_t index)
 
 extern "C" void list_reserve(Reference addr, size_t capacity)
 {
-    auto& list = (*reinterpret_cast<Type::List*>(addr.address));
+    auto& list = *reinterpret_cast<Type::List*>(addr.address);
     size_t inner = type_inner(addr.type);
     size_t offset = type_size(inner);
 
     rawlist_reserve(list.data, offset, capacity);
 }
 
-/*extern "C" void list_resize(Reference addr, size_t length)
-{
-    if(type_key(addr.type) == Type::Tag::List) {
-        //auto& list = (*reinterpret_cast<Type::List*>(addr.address));
 
+// Sparse //
+
+extern "C" size_t sparse_header_length(Reference addr)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    return list.header.length;
+}
+
+extern "C" Type::SparseHeader sparse_header_data(Reference addr, size_t index)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    Type::SparseHeader header {0};
+
+    if(index < list.header.length) {
+        header = *reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), index));
     }
-}*/
+
+    return header;
+}
+
+extern "C" size_t sparse_length(Reference addr)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    return list.data.length;
+}
+
+extern "C" void sparse_clear(Reference addr)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    size_t inner = type_inner(addr.type);
+
+    Reference ref {0};
+    ref.type = inner;
+    
+    for(size_t i = 0; i < list.data.length; ++i) {
+        ref.address = rawlist_cell(list.data, type_size(inner), i);
+        drop(ref);
+    }
+
+    rawlist_clear(list.data);
+    rawlist_clear(list.header);
+}
+
+extern "C" Reference sparse_at(Reference addr, size_t index)
+{
+    Reference result {0};
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    size_t inner = type_inner(addr.type);
+
+    if(index < list.data.length) {
+        auto cell = rawlist_cell(list.data, type_size(inner), index);
+        if(cell != nullptr) {
+            result.type = inner;
+            result.address = cell;
+        }
+
+        if(type_key(inner) == Type::Tag::Varying) {
+            result = varying_get(result);
+        }
+    }
+
+    return result;
+}
+
+extern "C" Reference sparse_get(Reference addr, size_t index)
+{
+    Reference result {0};
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    size_t inner = type_inner(addr.type);
+
+    size_t bound_lower = 0;
+    size_t bound_upper = list.header.length;
+    size_t header_index = 0;
+    Type::SparseHeader* header = nullptr;
+
+    while(bound_lower != bound_upper) {
+        header_index = bound_lower + ((bound_upper - bound_lower) / 2);
+        header = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), header_index));
+        
+        if(index >= header->start && index < (header->start + header->length)) {
+            break;
+        }
+        else if(index < header->start) {
+            bound_upper = header_index;
+        }
+        else {
+            bound_lower = header_index + 1;
+        }
+
+        header = nullptr;
+    }
+
+    if(header != nullptr) {
+        result.type = inner;
+        size_t data_index = header->index + (index - header->start);
+        result.address = rawlist_cell(list.data, type_size(inner), data_index);
+
+        if(type_key(result.type) == Type::Tag::Varying) {
+            result = varying_get(result);
+        }
+    }
+
+    return result;
+}
+
+extern "C" void sparse_set(Reference addr, size_t index, Reference source)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    size_t inner = type_inner(addr.type);
+
+    size_t header_index = 0;
+    Type::SparseHeader* header = nullptr;
+    for(; header_index < list.header.length; ++header_index) {
+        header = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), header_index));
+        if(index <= (header->start + header->length)) {
+            break;
+        }
+    }
+
+    Type::SparseHeader header_new;
+
+    Reference new_data {0};
+    new_data.type = inner;
+
+    if(header_index == list.header.length) {
+        // append header to end
+        header_new.start = index;
+        header_new.length = 1;
+        header_new.index = list.data.length;
+
+        new_data.address = rawlist_insert(list.data, type_size(inner), list.data.length);
+        copy(new_data, source);
+
+        auto& h_cell = *reinterpret_cast<Type::SparseHeader*>(rawlist_insert(list.header, sizeof(Type::SparseHeader), list.header.length));
+        h_cell = header_new;
+    } else {
+        if(index < header->start) {
+            // insert header before current
+            header_new.start = index;
+            header_new.length = 1;
+            header_new.index = header->index;
+
+            new_data.address = rawlist_insert(list.data, type_size(inner), header->index);
+            copy(new_data, source);
+
+            header = reinterpret_cast<Type::SparseHeader*>(rawlist_insert(list.header, sizeof(Type::SparseHeader), header_index));
+            *header = header_new;
+
+            for(size_t i = header_index + 1; i < list.header.length; ++i) {
+                auto hcell = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), i));
+                hcell->index++;
+            }
+        } else {
+            size_t offset = index - header->start;
+            if(offset < header->length) {
+                // update existing value
+                new_data.address = rawlist_cell(list.data, type_size(inner), header->index + offset);
+                drop(new_data);
+                copy(new_data, source);
+            } else {
+                // append value to current 
+                new_data.address = rawlist_insert(list.data, type_size(inner), header->index + offset);
+                copy(new_data, source);
+                header->length++;
+
+                for(size_t i = header_index + 1; i < list.header.length; ++i) {
+                    auto hcell = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), i));
+                    hcell->index++;
+                }
+            }
+        }
+
+        // join headers if ranges intersect
+        if(header_index + 1 < list.header.length) {
+            auto next_header = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), header_index + 1));
+            if(next_header->start == header->start + header->length) {
+                header->length += next_header->length;
+                rawlist_remove(list.header, sizeof(Type::SparseHeader), header_index + 1);
+            }
+        }
+    }
+}
+
+extern "C" void sparse_unset(Reference addr, size_t index)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+    size_t inner = type_inner(addr.type);
+
+    size_t header_index = 0;
+    Type::SparseHeader* header = nullptr;
+    for(; header_index < list.header.length; ++header_index) {
+        header = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), header_index));
+        if(index <= (header->start + header->length)) {
+            break;
+        }
+    }
+
+    Type::SparseHeader header_new;
+
+    Reference cell {0};
+    cell.type = inner;
+
+    if(header_index < list.header.length && index >= header->start) {
+        size_t offset = index - header->start;
+        size_t data_index = header->index + offset;
+        if(offset == 0) {
+            // shift start of range
+            header->start++;
+        } else if(offset < header->length - 1) {
+            // split header at index
+            header_new.start = index + 1;
+            header_new.length = header->length - offset - 1;
+            header_new.index = header->index + offset + 1;
+            header->length = offset + 1;
+
+            auto header_insert = reinterpret_cast<Type::SparseHeader*>(rawlist_insert(list.header, sizeof(Type::SparseHeader), header_index + 1));
+            *header_insert = header_new;
+        }
+
+        cell.address = rawlist_cell(list.data, type_size(inner), data_index);
+        drop(cell);
+        rawlist_remove(list.data, type_size(inner), data_index);
+        header->length--;
+
+        for(size_t i = header_index + 1; i < list.header.length; ++i) {
+            auto hcell = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), i));
+            hcell->index--;
+        }
+
+        if(header->length == 0) {
+            rawlist_remove(list.header, sizeof(Type::SparseHeader), header_index);
+        }
+    }
+}
+
+extern "C" size_t sparse_indexof(Reference addr, size_t index)
+{
+    auto& list = *reinterpret_cast<Type::Sparse*>(addr.address);
+
+    for(size_t i = 0; i < list.header.length; ++i) {
+        auto hcell = reinterpret_cast<Type::SparseHeader*>(rawlist_cell(list.header, sizeof(Type::SparseHeader), i));
+
+        if(index < hcell->length) {
+            return hcell->start + index;
+        } else {
+            index -= hcell->length;
+        }
+    }
+
+    return 0;
+}
 
 
 // Record //
@@ -833,7 +1063,8 @@ extern "C" Reference record_at(Reference addr, size_t index)
 extern "C" void record_update(Reference addr, size_t index, Reference source)
 {
     Reference destination {0};
-    auto binding = DB_SCHEMA.get(type_innerkey(addr.type));
+    size_t schid = type_innerkey(addr.type);
+    auto binding = DB_SCHEMA.get(schid);
     if(binding != nullptr) {
         if(index < binding->data.size()) {
             destination.type = binding->data[index].type;
@@ -960,7 +1191,7 @@ extern "C" void schema_map(Reference addr, size_t key, size_t index)
 extern "C" void schema_unmap(Reference addr, size_t key)
 {
     auto& object = (*reinterpret_cast<Type::Schema*>(addr.address));
-    for(size_t i; i < object.map.length; i++) {
+    for(size_t i; i < object.map.length; ++i) {
         auto cell = reinterpret_cast<Type::Schema::Mapping*>(rawlist_cell(object.map, sizeof(Type::Schema::Mapping), i));
         if(cell != nullptr) {
             if(cell->key == key) {
@@ -1036,7 +1267,7 @@ extern "C" size_t schema_bind(Reference addr, size_t id)
         printf("  Align: %zu\n", binding.alignment);
         printf("  Data:\n");
         for(size_t i = 0; i < binding.data.size(); ++i) {
-            printf("   - %zu {%#x} (%zu)\n",
+            printf("   - %zu:%#x +%zu\n",
                 binding.data[i].type,
                 type_key(binding.data[i].type),
                 binding.data[i].offset

src/runtime/lib.h

@@ -69,7 +69,7 @@ extern "C" size_t block_length(Reference addr);
 extern "C" uint8_t block_get(Reference addr, size_t index);
 extern "C" void block_set(Reference addr, size_t index, uint8_t value);
 
-// String //
+// Sequence //
 extern "C" size_t sequence_capacity(Reference addr);
 extern "C" size_t sequence_length(Reference addr);
 extern "C" uint8_t sequence_get(Reference addr, size_t index);
@@ -102,6 +102,17 @@ extern "C" void list_remove(Reference addr, size_t index);
 extern "C" void list_reserve(Reference addr, size_t capacity);
 //extern "C" void list_resize(Reference addr, size_t length);
 
+// Sparse //
+extern "C" size_t sparse_header_length(Reference addr);
+extern "C" Type::SparseHeader sparse_header_data(Reference addr, size_t index);
+extern "C" size_t sparse_length(Reference addr);
+extern "C" void sparse_clear(Reference addr);
+extern "C" Reference sparse_at(Reference addr, size_t index);
+extern "C" Reference sparse_get(Reference addr, size_t index);
+extern "C" void sparse_set(Reference addr, size_t index, Reference source);
+extern "C" void sparse_unset(Reference addr, size_t index);
+extern "C" size_t sparse_indexof(Reference addr, size_t index);
+
 // Record //
 extern "C" size_t record_length(Reference addr);
 extern "C" size_t record_type(Reference addr, size_t index);

src/runtime/mod.rs

@@ -28,6 +28,14 @@ pub struct Str {
     pub length:usize,
 }
 
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub struct SparseHeader {
+    pub start:usize,
+    pub length:usize,
+    pub index:usize,
+}
+
 extern "C" {
     pub fn test();
 
@@ -101,6 +109,16 @@ extern "C" {
     //pub fn list_find(addr:Reference, target:Reference, start:usize) -> usize;
     //pub fn list_count(addr:Reference, target:Reference, start:usize) -> usize;
 
+    pub fn sparse_header_length(addr:Reference) -> usize;
+    pub fn sparse_header_data(addr:Reference, index:usize) -> SparseHeader;
+    pub fn sparse_length(addr:Reference) -> usize;
+    pub fn sparse_clear(addr:Reference);
+    pub fn sparse_get(addr:Reference, index:usize) -> Reference;
+    pub fn sparse_at(addr:Reference, index:usize) -> Reference;
+    pub fn sparse_set(addr:Reference, index:usize, source:Reference);
+    pub fn sparse_unset(addr:Reference, index:usize);
+    pub fn sparse_indexof(addr:Reference, index:usize) -> usize;
+
     pub fn record_length(addr:Reference) -> usize;
     pub fn record_type(addr:Reference, index:usize) -> usize;
     pub fn record_at(addr:Reference, index:usize) -> Reference;

src/runtime/nametree.h

@@ -83,8 +83,8 @@ size_t NameTree::indexof(const std::string& key)
                     // create branching node if value is not prefix
                     if(index != key.length()) {
                         m_nodes.push_back(Node(key.substr(index), intermediate_index));
+                        m_nodes[intermediate_index].children.push_back(m_nodes.size() - 1);
                     }
-
                     return m_nodes.size() - 1;
                 }
             }
@@ -97,7 +97,6 @@ size_t NameTree::indexof(const std::string& key)
             return m_nodes.size() - 1;
         }
     }
-
     return current;
 }
 

src/runtime/rawlist.cc

@@ -1,6 +1,6 @@
 #include "rawlist.h"
 
-void* rawlist_insert(RawList& list, size_t offset, size_t index)
+uint8_t* rawlist_insert(RawList& list, size_t offset, size_t index)
 {
     if(list.length == list.capacity) { rawlist_reserve(list, offset, (list.capacity == 0) + list.capacity * 2); }
     if(index > list.length) { index = list.length; }

src/runtime/rawlist.h

@@ -11,7 +11,7 @@ struct RawList {
     uint8_t* data;
 };
 
-void* rawlist_insert(RawList& list, size_t offset, size_t index);
+uint8_t* rawlist_insert(RawList& list, size_t offset, size_t index);
 void rawlist_remove(RawList& list, size_t offset, size_t index);
 inline uint8_t* rawlist_cell(RawList& list, size_t offset, size_t index);
 void rawlist_reserve(RawList& list, size_t offset, size_t capacity);

src/runtime/sparselist.h

@@ -14,7 +14,7 @@ public:
 
     size_t length() const
     {
-
+        m_data.size();
     }
 
     void set(size_t index, const T& value)

src/runtime/type.h

@@ -23,7 +23,7 @@ namespace Tag {
         Sequence = 0x1f,
         Array    = 0x22,
         List     = 0x23,
-        //Sparse   = 0x24,
+        Sparse   = 0x24,
         Record   = 0x7e,
         Schema   = 0x7f,
     };
@@ -42,21 +42,15 @@ struct List {
     RawList data;
 };
 
+extern "C" struct SparseHeader {
+    size_t start;
+    size_t length;
+    size_t index;
+};
+
 struct Sparse {
-    struct Header {
-        size_t start;
-        size_t length;
-        size_t index;
-
-        Header() {
-            start = 0;
-            length = 0;
-            index = 0;
-        }
-    };
-
-    RawList data;
     RawList header;
+    RawList data;
 };
 
 struct Schema {

src/tag.rs

@@ -16,7 +16,7 @@ pub const SEQUENCE :usize = 0x1f;
 //pub const SET      :usize = 0x21;
 pub const ARRAY    :usize = 0x22;
 pub const LIST     :usize = 0x23;
-//pub const SPARSE   :usize = 0x24;
+pub const SPARSE   :usize = 0x24;
 //pub const TRIE     :usize = 0x25;
 //pub const MAP      :usize = 0x26;
 //pub const TREE     :usize = 0x27;