From 3a7975181eafe7f2691d1639a188f4d841f90dcc Mon Sep 17 00:00:00 2001 From: rhodey Date: Wed, 18 Feb 2026 14:47:06 -0500 Subject: [PATCH 1/8] support optimize other file types. and make faster. --- PRAGMA.md | 2 +- README.md | 15 +- schema.sql | 2 +- src/db/mod.rs | 40 +++++- src/fs/mod.rs | 363 +++++++++++++++++++++++++++--------------------- src/main.rs | 60 ++++---- test/6sql.js | 56 ++++++-- test/7umount.js | 21 +-- 8 files changed, 351 insertions(+), 208 deletions(-) diff --git a/PRAGMA.md b/PRAGMA.md index 3c20e42..7eeee01 100644 --- a/PRAGMA.md +++ b/PRAGMA.md @@ -7,7 +7,7 @@ All journal_modes are supported by SQLitesuperfs but only 2 are recommended. SQL DELETE is the SQLite default and it works fine. ## Journal_mode = TRUNCATE -TRUNCATE is at this time the SQLitesuperfs recommendation. It only runs maybe 5% faster than DELETE but it results in less dead tuples for the PSQL server. So it is better for the health of the server. +TRUNCATE is at this time the SQLitesuperfs recommendation. It runs faster than DELETE and it results in less dead tuples. ## Journal_mode = WAL Everyone wants to know about WAL. WAL is the crowd favorite and on a normal FS it is the best journal_mode. Many people think only WAL allows multi-reader but DELETE and TRUNCATE allow multi-reader also. The main advantage of WAL is that readers never block writes. With DELETE and TRUNCATE an INSERT *may* block waiting for a SELECT to complete. The SQLite locking protocol is actually very good and so there is only a small window in which for this to happen but it can happen. diff --git a/README.md b/README.md index ec78074..9f52f18 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,9 @@ All write operations are atomic (one PSQL txn) so this is good. However the linu SQLitesuperfs is at this time single threaded. SQLite is fundamentally 1-writer-multi-reader and so it does make sense for SQLitesuperfs to add a read thread pool. Contributions welcome! Keep in mind multiple apps are multiple namespaces (schemas) and do not have lock contention with eachother though they all share PSQL I/O. ## Quick Test -Nodejs is in this repo to do tests +Nodejs is in this repo to do tests: ``` +docker compose up -d psql mkdir -p ./testdir export psql_url=postgresql://psql:psql@localhost:5432/psql export encryption_pass=supersuper @@ -54,7 +55,7 @@ SQLitesuperfs passes `make test` in the official SQLite source tree. So you can: + Run `make test` from the mount ## Performance -Early testing is showing SQLite on SQLitesuperfs to be 2x to 3x slower when the PSQL server is localhost. When the PSQL server is eg Amazon Aurora in the same AZ SQLitesuperfs will be slower than 3x. SQLitesuperfs is primarily about *private hosted SQLite impossible => possible*. SQLitesuperfs even at 100x would be an achievement compared to Fully Homomorphic Encryption being 100,000x to 1,000,000x. +When the PSQL server is localhost SQLite is 15% faster than native FS thanks to SQLitesuperfs buffering. When the PSQL server has 0.2ms simulated RTT latency SQLite is 2x to 3x slower than native FS. SQLitesuperfs is primarily about *private hosted SQLite impossible => possible*. SQLitesuperfs even at 100x would be an achievement compared to Fully Homomorphic Encryption being 100,000x to 1,000,000x. The read thread pool described earlier will help and additionally a custom [VFS](https://sqlite.org/vfs.html) can be added to improve writes but TBH I am very happy already to build apps on this so I dont say I will be back here to add either *super* soon. @@ -67,7 +68,15 @@ The read thread pool described earlier will help and additionally a custom [VFS] ## Also Multi-Host Same-Namespace is not really on the roadmap but it is possible. Most of what you want from Multi-Host is: 1. durability 2. horizontal scaling. 1 is covered by PSQL and 2, well, just use a bigger host (and keep code simple). Multi-Host Same-Namespace would be something like LiteFS with SQLite WAL mode where you have 1 primary and N (asynchronous) replicas. LiteFS is cool but it should be noted that LiteFS durability is weaker, COMMIT always returns before replicas replicate. -One more doc [PRAGMA.md](PRAGMA.md) about SQLite `journal_mode` and `synchronous`. +## Also +SQLitesuperfs optimizations can be applied to data structures like [tinyraftplus](https://github.com/rhodey/tinyraftplus) if they have a lockfile protocol similar to SQLite: +``` +sqlitesuperfs super1 /tmp/super1 +sqlitesuperfs super1 /tmp/super1 --pattern db-journal,db +sqlitesuperfs super1 /tmp/super1 --pattern db-journal,db --pattern lock,off,log +``` + +The first and second commands are equivalent. And the third command is what would be used to optimize SQLite and tinyraftplus at the same time. One more doc [PRAGMA.md](PRAGMA.md) about SQLite `journal_mode` and `synchronous`. ## License mike@rhodey.org diff --git a/schema.sql b/schema.sql index 46a2799..3584d59 100644 --- a/schema.sql +++ b/schema.sql @@ -98,7 +98,7 @@ begin insert into :ns.blocks (ino, num, buf) values (i, n, b) on conflict (ino, num) do update set buf = b; - update :ns.inodes set size = sz where sz > 0 and id = i; + update :ns.inodes set size = sz where sz >= 0 and id = i; return 1; end; $$ language plpgsql; diff --git a/src/db/mod.rs b/src/db/mod.rs index 9d93fd8..c08c1b7 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,5 +1,6 @@ use log::debug; use log::error; +use log::{info, log_enabled, Level}; use std::fmt; use std::error::Error; @@ -24,6 +25,8 @@ use libsodium_rs::crypto_secretbox::Nonce; use crate::fs::Inode; use crate::fs::Block; +use std::{thread, time::Duration}; + pub struct PgDb { uid: u32, gid: u32, @@ -90,7 +93,7 @@ fn get_block(row: &Row) -> Block { ino: get_u64(row, "ino"), num: get_u64(row, "num"), buf: row.get("buf"), - ino_sz: 0, + ino_sz: -1, } } @@ -166,6 +169,11 @@ fn db_panic(op: &str, err: PgPretty) -> c_int { panic!("db panic ({}) {}", op, err); } +// simulate network +fn do_sleep() { + if log_enabled!(Level::Info) { thread::sleep(Duration::from_micros(200)); } +} + const SCHEMA: &str = include_str!("../../schema.sql"); impl PgDb { @@ -262,6 +270,8 @@ impl PgDb { } pub fn getattr(&mut self, ino: u64) -> Result, c_int> { + info!("getattr"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let query = format!("SELECT * FROM {}.inodes_and_links WHERE id = $1", &ns); @@ -278,6 +288,8 @@ impl PgDb { } pub fn lookup(&mut self, parent: u64, name: &str) -> Result, c_int> { + info!("lookup"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let parent: i64 = parent.try_into().unwrap(); @@ -294,6 +306,8 @@ impl PgDb { } pub fn readdir(&mut self, ino: u64, offset: i64) -> Result, c_int> { + info!("readdir"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let ino: i64 = ino.try_into().unwrap(); @@ -310,6 +324,8 @@ impl PgDb { } pub fn mknod(&mut self, parent: u64, name: &str, mode: u32, typee: u32, path: Option<&str>) -> Result { + info!("mknod"); + do_sleep(); let uid = self.uid; let gid = self.gid; let ns = self.namespace.to_string(); @@ -349,6 +365,8 @@ impl PgDb { } pub fn setattr(&mut self, inode: Inode, end_block: u64) -> Result { + info!("setattr"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let c: i64 = inode.create_ms.try_into().unwrap(); @@ -371,6 +389,8 @@ impl PgDb { } pub fn read(&mut self, ino: u64, start: u64, end: u64) -> Result, c_int> { + info!("read {}", ino); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let ino: i64 = ino.try_into().unwrap(); @@ -393,6 +413,7 @@ impl PgDb { } pub fn write(&mut self, blocks: Vec, attr: Option, end_block: u64) -> Result, c_int> { + info!("write"); let blocks: Vec = blocks.into_iter().map(|mut block| { let encrypted = self.encrypt(block.buf); block.buf = encrypted; @@ -401,6 +422,7 @@ impl PgDb { let ns = self.namespace.to_string(); let conn = self.get_conn()?; + do_sleep(); let mut txn = match conn.transaction() { Err(e) => return Err(db_err("write", PgPretty(e))), Ok(txn) => txn, @@ -418,6 +440,7 @@ impl PgDb { let f: i32 = attr.flags.try_into().unwrap(); let id: i64 = attr.id.try_into().unwrap(); let eb: i64 = end_block.try_into().unwrap(); + do_sleep(); let query = format!("SELECT * FROM {}.setattr($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", &ns); let row = match txn.query_one(&query, &[&id, &c, &m, &a, &sz, &mo, &u, &g, &f, &eb]) { Ok(row) => row, @@ -430,8 +453,10 @@ impl PgDb { for block in blocks { let ino: i64 = block.ino.try_into().unwrap(); let num: i64 = block.num.try_into().unwrap(); - let ino_sz: i64 = block.ino_sz.try_into().unwrap(); + let ino_sz: i64 = block.ino_sz; let buf = block.buf; + info!("write {} {}", ino, num); + do_sleep(); let query = format!("SELECT {}.write($1, $2, $3, $4) AS ok", &ns); match txn.query_one(&query, &[&ino, &num, &buf, &ino_sz]) { Err(e) => return Err(db_panic("write", PgPretty(e))), @@ -439,6 +464,7 @@ impl PgDb { }; } + do_sleep(); match txn.commit() { Err(e) => return Err(db_panic("write", PgPretty(e))), Ok(_) => {}, @@ -448,6 +474,8 @@ impl PgDb { } pub fn link(&mut self, ino: u64, new_parent: u64, new_name: &str) -> Result { + info!("link"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let query = format!("SELECT * FROM {}.link($1, $2, $3)", &ns); @@ -463,6 +491,8 @@ impl PgDb { } pub fn unlink(&mut self, parent: u64, name: &str) -> Result { + info!("unlink"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let query = format!("SELECT * FROM {}.unlink($1, $2)", &ns); @@ -477,6 +507,8 @@ impl PgDb { } pub fn rmdir(&mut self, parent: u64, name: &str) -> Result { + info!("rmdir"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let query = format!("SELECT * FROM {}.rmdir($1, $2) AS err", &ns); @@ -492,6 +524,8 @@ impl PgDb { } pub fn rename(&mut self, parent: u64, name: &str, new_parent: u64, new_name: &str) -> Result { + info!("rename"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let query = format!("SELECT * FROM {}.rename($1, $2, $3, $4)", &ns); @@ -506,6 +540,8 @@ impl PgDb { } pub fn del(&mut self, ino: u64) -> Result<(), c_int> { + info!("del"); + do_sleep(); let ns = self.namespace.to_string(); let conn = self.get_conn()?; let query = format!("DELETE FROM {}.inodes WHERE id = $1", &ns); diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 3171aeb..331b7c3 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,5 +1,4 @@ use log::debug; -// use log::info; use libc::c_int; use std::path::Path; @@ -54,7 +53,7 @@ pub struct Block { pub ino: u64, pub num: u64, pub buf: Vec, - pub ino_sz: u64, + pub ino_sz: i64, } fn round_up(n: u64, unit: u64) -> u64 { @@ -106,7 +105,7 @@ fn zero_pad(blocks: Vec, block_sz: u64, ino_sz: u64, first: u64) -> Vec; type Blocks = HashMap; type BlockCache = HashMap; +type Names = Vec>; #[allow(dead_code)] pub struct Fs { uid: libc::uid_t, gid: libc::gid_t, - block_sz: u64, buffers: u64, ttl: u64, - fhc: u64, + block_sz: u64, + buffer_bytes: u64, + ttl_ms: u64, fh_map: Cache, - fh_dmap: Cache, + fh_dir_map: Cache, + fh_next: u64, inodes: Cache, - dblocks: BlockCache, - jblocks: BlockCache, + names: Names, + buffers: BlockCache, + diffo: Blocks, pgdb: PgDb, } impl Fs { - pub fn new(uid: u32, gid: u32, block_sz: u64, buffers: u64, ttl: u64, pgdb: PgDb) -> Self { + pub fn new(uid: u32, gid: u32, block_sz: u64, buffers: u64, ttl: u64, names: Names, pgdb: PgDb) -> Self { + let buffer_bytes = 1024 * 1024 * buffers; + let ttl_ms = ttl * 1000; let fh_map: Cache = HashMap::new(); - let fh_dmap: Cache = HashMap::new(); + let fh_dir_map: Cache = HashMap::new(); let inodes: Cache = HashMap::new(); - let dblocks: BlockCache = HashMap::new(); - let jblocks: BlockCache = HashMap::new(); - let buffers = 1024 * 1024 * buffers; - let ttl = ttl * 1000; + let buffers: BlockCache = HashMap::new(); + let diffo: Blocks = HashMap::new(); Fs { uid, gid, - block_sz, buffers, ttl, - fhc: 0, fh_map, fh_dmap, inodes, - dblocks, jblocks, - pgdb, + block_sz, buffer_bytes, ttl_ms, + fh_map, fh_dir_map, fh_next: 0, + inodes, names, buffers, diffo, pgdb, } } + fn get_name_group(&mut self, name: OsString) -> Option> { + let name: String = name.into_string().unwrap_or_else(|_| "nothing.nothing".to_string()); + let ext = name.split('.').last().unwrap_or("nothing"); + for group in self.names.clone() { + for name in &group { + if name == ext { + return Some(group) + } + } + } + None + } + fn get_ino(&mut self, ino: u64) -> Result, c_int> { let _inode: Option = match self.inodes.get(&ino) { Some(inode) => return Ok(Some(inode.clone())), @@ -195,8 +210,8 @@ impl Fs { } fn trunc_buffers(&mut self, ino: u64, size: u64) -> bool { - if self.buffers == 0 { return false } - let block_map = self.dblocks.get_mut(&ino).or_else(|| self.jblocks.get_mut(&ino)); + if self.buffer_bytes == 0 { return false } + let block_map = self.buffers.get_mut(&ino); if block_map.is_none() { return false } let block_map = block_map.unwrap(); let end = div_ceil(size, self.block_sz); @@ -206,38 +221,44 @@ impl Fs { if let Some(last) = block_map.get_mut(&last) { last.buf.truncate(size as usize); } + let first = Block { ino, num: 0, buf: vec![], ino_sz: -1 }; + block_map.entry(0).or_insert_with(|| first); return true; } - // todo: ?? can copy more efficiently fn read_buffers(&mut self, ino: u64, start: u64, end: u64, full: bool) -> Result, c_int> { if full { + // writing full blocks so no read return Ok(Vec::new()); - } else if self.buffers == 0 { + } else if self.buffer_bytes == 0 || self.buffers.get(&ino).is_none() { + // buffers disabled match self.pgdb.read(ino, start, end) { Ok(blocks) => return Ok(blocks), Err(errno) => return Err(errno), }; } + + // copy let mut blocks: Vec = Vec::new(); - let block_map = self.dblocks.get(&ino).or_else(|| self.jblocks.get(&ino)); - if let Some(block_map) = block_map { - for num in start..end { - if let Some(block) = block_map.get(&num) { - blocks.push(block.clone()); - } + let block_map = self.buffers.get(&ino).unwrap(); + for num in start..end { + if let Some(block) = block_map.get(&num) { + blocks.push(block.clone()); } } + if blocks.len() >= ((end - start) as usize) { return Ok(blocks) } + + // need more let also = match self.pgdb.read(ino, start, end) { Ok(blocks) => blocks, Err(errno) => return Err(errno), }; + + // copy let also: Blocks = also.into_iter().map(|block| (block.num, block)).collect(); for num in start..end { - if let Some(block_map) = block_map { - if block_map.contains_key(&num) { continue } - } + if block_map.contains_key(&num) { continue } if let Some(also) = also.get(&num) { blocks.push(also.clone()); } @@ -245,83 +266,95 @@ impl Fs { Ok(blocks) } - // todo: ?? can copy more efficiently - fn sync_buffers(&mut self, ino: u64, db: bool, journal: bool) -> bool { - if self.buffers == 0 { return false } + fn sync_buffers(&mut self, ino: u64) -> bool { + // disabled + if self.buffer_bytes == 0 { return false } let ino1 = match self.get_ino(ino) { Ok(Some(inode)) => inode, Ok(None) => return false, Err(errno) => { panic!("(sync_buffers) (get_ino) {}", errno); }, }; - let single = db == false && journal == false; - let name; - if let Some(n) = ino1.name.to_str() { - if n.ends_with(".db") && (db || single) { - name = n; - } else if n.ends_with(".db-journal") && (journal || single) { - name = n; - } else { - return false; - } - } else { - return false; - } - - let mut blocks1: Vec = Vec::new(); - let block_map = self.dblocks.get_mut(&ino).or_else(|| self.jblocks.get_mut(&ino)); - if block_map.is_none() { return false } - let block_map = block_map.unwrap(); - for block in block_map.values() { blocks1.push(block.clone()); } - if let Some(mut last) = blocks1.pop() { - last.ino_sz = ino1.size; - blocks1.push(last); + let group1 = self.get_name_group(ino1.name.clone()); + // no --pattern arg matches name + if group1.is_none() { return false } + + let group1 = group1.unwrap(); + let keys: Vec = self.buffers.keys().copied().collect(); + let mut all: Vec = Vec::new(); + + for ino in keys { + match self.get_ino(ino) { + Ok(Some(inode)) => all.push(inode), + Err(errno) => { panic!("(sync_buffers) (get_ino 2) {}", errno); }, + _ => (), + }; } - let keys = if name.ends_with(".db-journal") { - block_map.clear(); - self.dblocks.keys() - } else { - block_map.retain(|&key, _| key == 0); - self.jblocks.keys() - }; + // in same dir + let all: Vec = all + .into_iter() + .filter(|ino2| ino2.parent == ino1.parent) + .collect(); - let test = if name.ends_with(".db-journal") { - name.replace("-journal", "") - } else { - format!("{}-journal", name) - }; + // in same group + let all: Vec = all + .into_iter() + .filter(|ino2| { + let name1: String = ino1.name.clone().into_string().unwrap_or_else(|_| "nothing.nothing".to_string()); + let name1 = name1.split('.').next().unwrap_or("nothing"); + let name2: String = ino2.name.clone().into_string().unwrap_or_else(|_| "nothing.nothing".to_string()); + let ext2 = name2.split('.').last().unwrap_or("nothing"); + let name2 = name2.split('.').next().unwrap_or("nothing"); + group1.contains(&ext2.to_string()) && name1 == name2 + }) + .collect(); - let mut i2: Option = None; - let mut blocks2: Vec = Vec::new(); - if single == false { - for ino in keys { - let block_map = self.dblocks.get(&ino).or_else(|| self.jblocks.get(&ino)).unwrap(); - if block_map.len() == 0 { continue } - if let Some(ino2) = self.inodes.get(ino) { - if ino2.parent != ino1.parent { continue } - if let Some(name2) = ino2.name.to_str() { - if test != name2 { continue } - i2 = Some(ino2.id); - for block in block_map.values() { blocks2.push(block.clone()); } - if let Some(mut last) = blocks2.pop() { - last.ino_sz = ino2.size; - blocks2.push(last); + // copy blocks + let mut copy: Vec = Vec::new(); + for ino in all { + let block_map = self.buffers.get_mut(&ino.id); + if block_map.is_none() { continue } + let block_map = block_map.unwrap(); + let mut copyy: Vec = Vec::new(); + + if block_map.len() > 1 { + for block in block_map.values() { + copyy.push(block.clone()); + } + if let Some(mut last) = copyy.pop() { + // last block sets size in db + last.ino_sz = ino.size.try_into().unwrap(); + copyy.push(last); + } + } else { + // possibly nothing changed + if let Some(first) = block_map.get(&0) { + if let Some(prev) = self.diffo.get(&ino.id) { + if first.buf != prev.buf { + let mut first = first.clone(); + first.ino_sz = ino.size.try_into().unwrap(); + copyy.push(first); } + } else { + let mut first = first.clone(); + first.ino_sz = ino.size.try_into().unwrap(); + copyy.push(first); } + } else { + panic!("(sync_buffers) (0 None)"); } } - } - if let Some(i2) = i2 { - if name.ends_with(".db-journal") { - self.dblocks.get_mut(&i2).unwrap().retain(|&key, _| key == 0); - } else { - self.jblocks.get_mut(&i2).unwrap().clear(); - } + copy.extend(copyy); + block_map.retain(|&key, _| key == 0); + let first = block_map.get(&0).unwrap(); + self.diffo.insert(ino.id, first.clone()); } - let blocks = [blocks1, blocks2].concat(); - match self.pgdb.write(blocks, None, 0) { + // nothing changed + if copy.len() == 0 { return true } + + match self.pgdb.write(copy, None, 0) { Err(errno) => { panic!("(sync_buffers) (write) {}", errno); }, Ok(_) => return true, }; @@ -335,7 +368,6 @@ impl Filesystem for Fs { Ok(()) } - // fuse-rs never calls fn destroy(&mut self, _req: &Request) { debug!("destroy"); } @@ -359,7 +391,7 @@ impl Filesystem for Fs { Ok(None) => return reply.error(libc::ENOENT), Err(errno) => return reply.error(errno), }; - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); reply.attr(&ttl, &inode.attr(self.block_sz)); } @@ -370,7 +402,7 @@ impl Filesystem for Fs { Ok(None) => return reply.error(libc::ENOENT), Err(errno) => return reply.error(errno), }; - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); let gen = 0; reply.entry(&ttl, &inode.attr(self.block_sz), gen); } @@ -384,15 +416,15 @@ impl Filesystem for Fs { }; inode.open += 1; self.inodes.insert(ino, inode.clone()); - self.fhc += 1; - let fh = self.fhc; - self.fh_dmap.insert(fh, inode); + self.fh_next += 1; + let fh = self.fh_next; + self.fh_dir_map.insert(fh, inode); reply.opened(fh, flags); } fn readdir(&mut self, _req: &Request, ino: u64, fh: u64, offset: i64, mut reply: ReplyDirectory) { if offset < 0 { return reply.error(libc::EINVAL) } - let _inode = match self.fh_dmap.get(&fh) { + let _inode = match self.fh_dir_map.get(&fh) { Some(inode) if inode.id == ino => inode, _ => return reply.error(libc::EBADF), }; @@ -443,7 +475,7 @@ impl Filesystem for Fs { } fn releasedir(&mut self, _req: &Request, ino: u64, fh: u64, _flags: u32, reply: ReplyEmpty) { - let _inode = match self.fh_dmap.get(&fh) { + let _inode = match self.fh_dir_map.get(&fh) { Some(inode) if inode.id == ino => inode, _ => return reply.error(libc::EBADF), }; @@ -454,7 +486,7 @@ impl Filesystem for Fs { }; inode.open = inode.open.saturating_sub(1); self.inodes.insert(ino, inode.clone()); - self.fh_dmap.remove(&fh); + self.fh_dir_map.remove(&fh); if inode.nlink > 0 || inode.open > 0 { return reply.ok(); } self.inodes.remove(&ino); match self.pgdb.del(ino) { @@ -470,7 +502,7 @@ impl Filesystem for Fs { Err(errno) => return reply.error(errno), }; self.inodes.insert(inode.id, inode.clone()); - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); let gen = 0; reply.entry(&ttl, &inode.attr(self.block_sz), gen); } @@ -488,22 +520,23 @@ impl Filesystem for Fs { } inode.open += 1; self.inodes.insert(ino, inode.clone()); - self.fhc += 1; - let fh = self.fhc; + self.fh_next += 1; + let fh = self.fh_next; self.fh_map.insert(fh, inode.clone()); if let Some(name) = inode.name.to_str() { - if self.buffers > 0 && name.ends_with(".db") { - self.dblocks.entry(ino).or_insert_with(|| HashMap::new()); - let blocks = self.dblocks.get_mut(&ino).unwrap(); - if blocks.contains_key(&0) { return reply.opened(fh, flags); } + if self.buffer_bytes > 0 && self.get_name_group(name.into()).is_some() { + self.buffers.entry(ino).or_insert_with(|| HashMap::new()); + let blocks = self.buffers.get_mut(&ino).unwrap(); + // always keep block 0 in buffers + if blocks.contains_key(&0) { + return reply.opened(fh, flags); + } let first = match self.pgdb.read(ino, 0, 1) { Ok(blocks) => blocks, Err(errno) => return reply.error(errno), }; - let first = first.first().cloned().unwrap_or(Block { ino, num: 0, buf: vec![], ino_sz: 0 }); + let first = first.first().cloned().unwrap_or(Block { ino, num: 0, buf: vec![], ino_sz: -1 }); blocks.insert(0, first); - } else if self.buffers > 0 && name.ends_with(".db-journal") { - self.jblocks.entry(ino).or_insert_with(|| HashMap::new()); } } reply.opened(fh, flags); @@ -517,20 +550,19 @@ impl Filesystem for Fs { }; inode.open += 1; self.inodes.insert(inode.id, inode.clone()); - self.fhc += 1; - let fh = self.fhc; + self.fh_next += 1; + let fh = self.fh_next; self.fh_map.insert(fh, inode.clone()); if let Some(name) = inode.name.to_str() { - if self.buffers > 0 && name.ends_with(".db") { - let first = Block { ino: inode.id, num: 0, buf: vec![], ino_sz: 0 }; + if self.buffer_bytes > 0 && self.get_name_group(name.into()).is_some() { + // always keep block 0 in buffers + let first = Block { ino: inode.id, num: 0, buf: vec![], ino_sz: -1 }; let mut blocks = HashMap::new(); blocks.insert(0, first); - self.dblocks.insert(inode.id, blocks); - } else if self.buffers > 0 && name.ends_with(".db-journal") { - self.jblocks.insert(inode.id, HashMap::new()); + self.buffers.insert(inode.id, blocks); } } - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); let gen = 0; reply.created(&ttl, &inode.attr(self.block_sz), gen, fh, flags); } @@ -610,7 +642,9 @@ impl Filesystem for Fs { Ok(Some(inode)) => inode, _ => return reply.error(libc::EIO), }; + if data.is_empty() { return reply.written(0); } + let offset = offset.max(0) as u64; let sz = data.len() as u64; let block_sz = self.block_sz; @@ -619,14 +653,17 @@ impl Filesystem for Fs { let ino_sz = orig.size; let next_sz = orig.size.max(offset + sz); - // if writing full blocks then there is no need to read + // if writing full blocks then no need to read let full_blocks = offset == (start * block_sz) && (offset + sz) == (end * block_sz); // read from buffers and/or db - let mut blocks = match self.read_buffers(ino, start, end, full_blocks) { - Ok(blocks) => blocks, - Err(errno) => return reply.error(errno), - }; + let mut blocks: Vec = Vec::new(); + if start * block_sz < ino_sz { + blocks = match self.read_buffers(ino, start, end, full_blocks) { + Ok(blocks) => blocks, + Err(errno) => return reply.error(errno), + }; + } // write blocks.sort_by_key(|block| block.num); @@ -671,25 +708,28 @@ impl Filesystem for Fs { let block = &ready[off..end]; let block = block.to_vec(); off += block.len(); - blocks.push(Block { ino, num: num, buf: block, ino_sz: 0 }); + blocks.push(Block { ino, num: num, buf: block, ino_sz: -1 }); num += 1; } - // to txn buffer - if self.buffers > 0 { - let s1: usize = self.dblocks.values().map(|map| map.len()).sum(); - let s2: usize = self.jblocks.values().map(|map| map.len()).sum(); - let mut size = ((s1 + s2) as u64) * self.block_sz; - if let Some(block_map) = self.dblocks.get_mut(&ino).or_else(|| self.jblocks.get_mut(&ino)) { + // buffers + if self.buffer_bytes > 0 { + let total: usize = self.buffers.values().map(|map| map.len()).sum(); + let mut total = (total as u64) * self.block_sz; + if let Some(block_map) = self.buffers.get_mut(&ino) { for block in blocks { if block_map.insert(block.num, block).is_none() { - size += self.block_sz; + // was new entry + total += self.block_sz; } } orig.size = next_sz; self.inodes.insert(orig.id, orig); - if size <= self.buffers { return reply.written(sz as u32); } - if self.sync_buffers(ino, true, true) { + if total <= self.buffer_bytes { + // no sync + return reply.written(sz as u32); + } else if self.sync_buffers(ino) { + // sync return reply.written(sz as u32); } else { panic!("(write) (sync_buffers) false"); @@ -697,8 +737,9 @@ impl Filesystem for Fs { } } + // no buffers let mut last = blocks.pop().unwrap(); - last.ino_sz = next_sz; + last.ino_sz = next_sz.try_into().unwrap(); blocks.push(last); // to db @@ -751,11 +792,20 @@ impl Filesystem for Fs { inode.flags = flags.unwrap(); } - // is buffered - if size.is_some() && self.trunc_buffers(ino, inode.size) && inode.size > 0 { + let is_buffered = self.trunc_buffers(ino, inode.size); + + if size == Some(0) && is_buffered { + // is buffer commit + self.sync_buffers(ino); orig.size = inode.size; self.inodes.insert(orig.id, orig.clone()); - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); + return reply.attr(&ttl, &orig.attr(self.block_sz)); + } else if size.is_some() && is_buffered && inode.size > 0 { + // is buffer size change + orig.size = inode.size; + self.inodes.insert(orig.id, orig.clone()); + let ttl = to_ts(self.ttl_ms); return reply.attr(&ttl, &orig.attr(self.block_sz)); } @@ -763,13 +813,14 @@ impl Filesystem for Fs { let end_block = div_ceil(inode.size, block_sz); if size.is_none() || size == Some(orig.size) || (inode.size % block_sz) == 0 || inode.size > orig.size { - // is fast (normal) + // is common (fast) inode = match self.pgdb.setattr(inode, end_block) { Ok(inode) => inode, Err(errno) => return reply.error(errno), }; } else { - // is slow (rare) + // is rare (slower) + // size < orig.size && not % block_sz && not buffered let last_block = end_block.saturating_sub(1); let last_len = inode.size.saturating_sub(last_block * block_sz) as u32; let blocks = match self.pgdb.read(ino, last_block, end_block) { @@ -793,7 +844,7 @@ impl Filesystem for Fs { inode.open = orig.open; self.inodes.insert(inode.id, inode.clone()); - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); reply.attr(&ttl, &inode.attr(self.block_sz)); } @@ -802,7 +853,7 @@ impl Filesystem for Fs { Some(inode) if inode.id == ino => true, _ => false, }; - let dir = match self.fh_dmap.get(&fh) { + let dir = match self.fh_dir_map.get(&fh) { Some(inode) if inode.id == ino => true, _ => false, }; @@ -817,19 +868,11 @@ impl Filesystem for Fs { Some(inode) if inode.id == ino => ino, _ => return reply.error(libc::EBADF), }; - if let Some(_block_map) = self.dblocks.get(&ino) { - if self.sync_buffers(ino, true, false) { - return reply.ok(); - } else { - panic!("(fsync) (sync_buffers) false"); - } - } else { - return reply.ok(); - } + return reply.ok(); } fn fsyncdir(&mut self, _req: &Request, ino: u64, fh: u64, _datasync: bool, reply: ReplyEmpty) { - let _inode = match self.fh_dmap.get(&fh) { + let _inode = match self.fh_dir_map.get(&fh) { Some(inode) if inode.id == ino => return reply.ok(), _ => return reply.error(libc::EBADF), }; @@ -848,11 +891,11 @@ impl Filesystem for Fs { inode.open = inode.open.saturating_sub(1); self.inodes.insert(ino, inode.clone()); self.fh_map.remove(&fh); - if inode.open == 0 { self.sync_buffers(ino, false, false); } + if inode.open == 0 { self.sync_buffers(ino); } if inode.nlink > 0 || inode.open > 0 { return reply.ok(); } self.inodes.remove(&ino); - self.dblocks.remove(&ino); - self.jblocks.remove(&ino); + self.buffers.remove(&ino); + self.diffo.remove(&ino); match self.pgdb.del(ino) { Ok(_) => reply.ok(), Err(errno) => reply.error(errno), @@ -875,8 +918,8 @@ impl Filesystem for Fs { self.inodes.insert(inode.id, inode.clone()); if inode.nlink > 0 || inode.open > 0 { return reply.ok(); } self.inodes.remove(&inode.id); - self.dblocks.remove(&inode.id); - self.jblocks.remove(&inode.id); + self.buffers.remove(&inode.id); + self.diffo.remove(&inode.id); match self.pgdb.del(inode.id) { Ok(_) => reply.ok(), Err(errno) => reply.error(errno), @@ -897,7 +940,7 @@ impl Filesystem for Fs { parent.nlink += 1; self.inodes.insert(inode.id, inode.clone()); self.inodes.insert(parent.id, parent); - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); let gen = 0; reply.entry(&ttl, &inode.attr(self.block_sz), gen); } @@ -945,7 +988,7 @@ impl Filesystem for Fs { }; inode.open = orig.open; self.inodes.insert(ino, inode.clone()); - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); let gen = 0; reply.entry(&ttl, &inode.attr(self.block_sz), gen); } @@ -960,7 +1003,7 @@ impl Filesystem for Fs { Err(errno) => return reply.error(errno), }; self.inodes.insert(inode.id, inode.clone()); - let ttl = to_ts(self.ttl); + let ttl = to_ts(self.ttl_ms); let gen = 0; reply.entry(&ttl, &inode.attr(self.block_sz), gen); } diff --git a/src/main.rs b/src/main.rs index 7b58ad9..b857460 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ pub mod fs; pub mod db; use clap::Arg; +use clap::ArgAction; use std::path::Path; use libsodium_rs::{self, ensure_init}; @@ -26,6 +27,7 @@ fn main() { .arg(Arg::new("uid").short('u').value_parser(clap::value_parser!(u32))) .arg(Arg::new("gid").short('g').value_parser(clap::value_parser!(u32))) .arg(Arg::new("kernel_opts").short('o')) + .arg(Arg::new("pattern").long("pattern").short('p').action(ArgAction::Append).num_args(1)) .get_matches(); let namespace = matches.get_one::("namespace").unwrap(); @@ -67,31 +69,41 @@ fn main() { let kernel_opts = format!("{},{},{}", u, g, kernel_opts); debug!("{} kernel_opts", kernel_opts); - if Path::new(&mount_dir).exists() { - let umount_cmd = format!("fusermount -u {}", mount_dir); - ctrlc::set_handler(move || { - println!("signal = unmount"); - std::process::Command::new("sh") - .arg("-c") - .arg(&umount_cmd) - .output() - .expect("error fusermount"); - std::process::exit(0); - }) - .expect("error signals"); - - let mut pgdb = db::PgDb::new(uid, gid, &psql_url, namespace, &enc_pass); - let block_szz = pgdb.init(sql_schema, block_sz).expect("error init_schema"); - - if block_sz != block_szz { - eprintln!("block size {} does not match db block size {}", block_sz, block_szz); - std::process::exit(1); - } - - let fs = fs::Fs::new(uid, gid, block_sz, buffers, ttl, pgdb); - fs::mount(fs, mount_dir, &kernel_opts); - } else { + let patterns: Vec = matches + .get_many::("pattern") + .map(|vals| vals.cloned().collect()) + .unwrap_or_else(|| vec!["db-journal,db".to_string()]); + let patterns: Vec> = patterns + .into_iter() + .map(|p| { p.split(',').map(|s| s.trim().to_string()).collect() }) + .collect(); + debug!("{:?} patterns", patterns); + + if Path::new(&mount_dir).exists() == false { error!("mount_dir {} does not exist", mount_dir); std::process::exit(1); } + + let umount_cmd = format!("fusermount -u {}", mount_dir); + ctrlc::set_handler(move || { + println!("signal = unmount"); + std::process::Command::new("sh") + .arg("-c") + .arg(&umount_cmd) + .output() + .expect("error fusermount"); + std::process::exit(0); + }) + .expect("error signals"); + + let mut pgdb = db::PgDb::new(uid, gid, &psql_url, namespace, &enc_pass); + let block_szz = pgdb.init(sql_schema, block_sz).expect("error init_schema"); + + if block_sz != block_szz { + eprintln!("block size {} does not match db block size {}", block_sz, block_szz); + std::process::exit(1); + } + + let fs = fs::Fs::new(uid, gid, block_sz, buffers, ttl, patterns, pgdb); + fs::mount(fs, mount_dir, &kernel_opts); } diff --git a/test/6sql.js b/test/6sql.js index c0eab44..704599c 100644 --- a/test/6sql.js +++ b/test/6sql.js @@ -152,6 +152,42 @@ test('sql big txn', (t) => { t.end() }) +test('sql two txn', (t) => { + empty(DIR) + + const file = `${DIR}/test.db` + const db = Database(file) + db.pragma('journal_mode = TRUNCATE') + db.pragma('synchronous = FULL') + db.exec(table) + + let users = new Array(200).fill(0) + users = users.map((z, idx) => { + const uname = `u${idx}` + const email = `e${idx}` + return { uname, email } + }) + + const stmt = db.prepare('insert into users (uname, email) values (@uname, @email)') + const insertMany = db.transaction((users) => { + for (const user of users) { + stmt.run(user) + } + }) + + for (let c = 0; c < 4; c += 2) { + const two = users.slice(c, c + 2) + insertMany(two) + } + t.pass(`insert ok`) + + const row = db.prepare('select count(id) as c from users').get() + t.equal(row.c, 4, 'count ok') + + db.close() + t.end() +}) + test('sql many txn', (t) => { empty(DIR) @@ -177,19 +213,21 @@ test('sql many txn', (t) => { } }) - for (let i = 0; i < users.length; i += 2) { - const two = users.slice(i, i + 2) - insertMany(two) + let ms = 0 + for (let i = 0; i < 10; i++) { + db.prepare('delete from users').run() + begin = Date.now() + for (let c = 0; c < users.length; c += 2) { + const two = users.slice(c, c + 2) + insertMany(two) + } + ms = Date.now() - begin + t.pass(`insert ok ${i}`) + t.pass(`insert ${i} ${ms}ms`) } - let ms = Date.now() - begin - t.pass('insert ok') - t.pass(`insert ${ms}ms`) - begin = Date.now() const row = db.prepare('select count(id) as c from users').get() - ms = Date.now() - begin t.equal(row.c, users.length, 'count ok') - t.pass(`count ${ms}ms`) db.close() t.end() diff --git a/test/7umount.js b/test/7umount.js index 66c6755..c1d87b4 100644 --- a/test/7umount.js +++ b/test/7umount.js @@ -128,11 +128,11 @@ async function testInsert(t, mode, sig) { t.end() } -test('test insert DELETE SIGINT', (t) => testInsert(t, 'DELETE', 'SIGINT')) -test('test insert TRUNCATE SIGINT', (t) => testInsert(t, 'TRUNCATE', 'SIGINT')) +test('insert DELETE SIGINT', (t) => testInsert(t, 'DELETE', 'SIGINT')) +test('insert TRUNCATE SIGINT', (t) => testInsert(t, 'TRUNCATE', 'SIGINT')) -test('test insert DELETE SIGTERM', (t) => testInsert(t, 'DELETE', 'SIGTERM')) -test('test insert TRUNCATE SIGTERM', (t) => testInsert(t, 'TRUNCATE', 'SIGTERM')) +test('insert DELETE SIGTERM', (t) => testInsert(t, 'DELETE', 'SIGTERM')) +test('insert TRUNCATE SIGTERM', (t) => testInsert(t, 'TRUNCATE', 'SIGTERM')) async function testUpdate(t, mode, sig) { let ended = false @@ -198,8 +198,13 @@ async function testUpdate(t, mode, sig) { t.end() } -test('test update DELETE SIGINT', (t) => testUpdate(t, 'DELETE', 'SIGINT')) -test('test update TRUNCATE SIGINT', (t) => testUpdate(t, 'TRUNCATE', 'SIGINT')) +test('update DELETE SIGINT', (t) => testUpdate(t, 'DELETE', 'SIGINT')) +test('update TRUNCATE SIGINT', (t) => testUpdate(t, 'TRUNCATE', 'SIGINT')) -test('test update DELETE SIGTERM', (t) => testUpdate(t, 'DELETE', 'SIGTERM')) -test('test update TRUNCATE SIGTERM', (t) => testUpdate(t, 'TRUNCATE', 'SIGTERM')) +test('update DELETE SIGTERM', (t) => testUpdate(t, 'DELETE', 'SIGTERM')) +test('update TRUNCATE SIGTERM', (t) => testUpdate(t, 'TRUNCATE', 'SIGTERM')) + +test('reset', async (t) => { + await reset() + t.pass('reset ok') +}) From 2ca35f22a03a5316a17f53541171b51b442f2cda Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 11:09:18 -0500 Subject: [PATCH 2/8] README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9f52f18..7d807e2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # SQLitesuperfs -FUSE fs with PostgreSQL backend, block-level encryption, and optimizations for SQLite multi-tenancy. +SQLite FUSE fs with PostgreSQL backend, block-level encryption, and optimizations for multi-tenancy. ## Why [Lock.host](https://github.com/rhodey/lock.host) allows apps to attest to their code and to encrypt comms with clients. Lock.host also allows apps to create persistent keys. SQLitesuperfs was created to allow apps to keep persistent state (SQLite) and to keep that state private. All existing open-source encrypted filesystems use AES which fundamentally is vulnerable to timing attacks. Timing attacks mean "do not run this in the cloud". [Libsodium](https://doc.libsodium.org/) is used and is not vulnerable to timing attacks. @@ -66,17 +66,17 @@ The read thread pool described earlier will help and additionally a custom [VFS] + (Every encrypted filesystem is vulnerable to rollback) ## Also -Multi-Host Same-Namespace is not really on the roadmap but it is possible. Most of what you want from Multi-Host is: 1. durability 2. horizontal scaling. 1 is covered by PSQL and 2, well, just use a bigger host (and keep code simple). Multi-Host Same-Namespace would be something like LiteFS with SQLite WAL mode where you have 1 primary and N (asynchronous) replicas. LiteFS is cool but it should be noted that LiteFS durability is weaker, COMMIT always returns before replicas replicate. - -## Also -SQLitesuperfs optimizations can be applied to data structures like [tinyraftplus](https://github.com/rhodey/tinyraftplus) if they have a lockfile protocol similar to SQLite: +SQLitesuperfs optimizations can be applied to data structures like [tinyraftplus](https://github.com/rhodey/tinyraftplus) if they have a lockfile protocol similar to SQLite. The first and second commands are equivalent. And the third command is what would be used to optimize SQLite and tinyraftplus at the same time: ``` sqlitesuperfs super1 /tmp/super1 sqlitesuperfs super1 /tmp/super1 --pattern db-journal,db sqlitesuperfs super1 /tmp/super1 --pattern db-journal,db --pattern lock,off,log ``` -The first and second commands are equivalent. And the third command is what would be used to optimize SQLite and tinyraftplus at the same time. One more doc [PRAGMA.md](PRAGMA.md) about SQLite `journal_mode` and `synchronous`. +## Also +Multi-Host Same-Namespace is not really on the roadmap but it is possible. Most of what you want from Multi-Host is: 1. durability 2. horizontal scaling. 1 is covered by PSQL and 2, well, just use a bigger host (and keep code simple). Multi-Host Same-Namespace would be something like LiteFS with SQLite WAL mode where you have 1 primary and N (asynchronous) replicas. LiteFS is cool but it should be noted that LiteFS durability is weaker, COMMIT always returns before replicas replicate. + +One more doc [PRAGMA.md](PRAGMA.md) about SQLite `journal_mode` and `synchronous`. ## License mike@rhodey.org From 71cb6a63d877eea2fa2fd1f66a3f501a445809d4 Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 11:22:22 -0500 Subject: [PATCH 3/8] first raft test --- package-lock.json | 197 +++++++++++++++++++++++++++++++- package.json | 5 +- test/7raft.js | 82 +++++++++++++ test/{7umount.js => 8umount.js} | 0 4 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 test/7raft.js rename test/{7umount.js => 8umount.js} (100%) diff --git a/package-lock.json b/package-lock.json index 58b33a0..884df78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "devDependencies": { "better-sqlite3": "^12.4.6", "split": "^1.0.1", - "tape": "^5.9.0" + "tape": "^5.9.0", + "tinyraftplus": "^0.3.0" } }, "node_modules/@ljharb/resumer": { @@ -41,6 +42,90 @@ "node": ">= 0.4" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -278,6 +363,13 @@ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", "dev": true }, + "node_modules/combinations": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/combinations/-/combinations-1.0.0.tgz", + "integrity": "sha512-aVgTfI/dewHblSn4gF+NZHvS7wtwg9YAPF2EknHMdH+xLsXLLIMpmHkSj64Zxs/R2m9VAAgn3bENjssrn7V4vQ==", + "dev": true, + "license": "MIT" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -953,6 +1045,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hash-wasm": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.12.0.tgz", + "integrity": "sha512-+/2B2rYLb48I/evdOIhP+K/DD2ca2fgBjp6O+GBEnCDk2e4rpeXIK8GvIyRPjTezgmWn9gmKwkQjjx6BtqDHVQ==", + "dev": true, + "license": "MIT" + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -1408,6 +1507,23 @@ "dev": true, "license": "MIT" }, + "node_modules/libsodium": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/libsodium/-/libsodium-0.8.2.tgz", + "integrity": "sha512-TsnGYMoZtpweT+kR+lOv5TVsnJ/9U0FZOsLFzFOMWmxqOAYXjX3fsrPAW+i1LthgDKXJnI9A8dWEanT1tnJKIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/libsodium-wrappers": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/libsodium-wrappers/-/libsodium-wrappers-0.8.2.tgz", + "integrity": "sha512-VFLmfxkxo+U9q60tjcnSomQBRx2UzlRjKWJqvB4K1pUqsMQg4cu3QXA2nrcsj9A1qRsnJBbi2Ozx1hsiDoCkhw==", + "dev": true, + "license": "ISC", + "dependencies": { + "libsodium": "^0.8.0" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -1452,6 +1568,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -1480,6 +1612,39 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/msgpackr": { + "version": "1.11.8", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.8.tgz", + "integrity": "sha512-bC4UGzHhVvgDNS7kn9tV8fAucIYUBuGojcaLiz7v+P63Lmtm0Xeji8B/8tYKddALXxJLpwIeBmUN3u64C4YkRA==", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/napi-build-utils": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", @@ -1498,6 +1663,22 @@ "node": ">=10" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -2185,6 +2366,20 @@ "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "dev": true }, + "node_modules/tinyraftplus": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/tinyraftplus/-/tinyraftplus-0.3.0.tgz", + "integrity": "sha512-hoe54eBUemlRqY3+Tg8euQmixv+Ht3yY+ImOxKIL8h2TDSHcmJgxszmMxb9vmKFqMQuRgG0Ib7c8JAPn1p5jXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "combinations": "^1.0.0", + "hash-wasm": "^4.12.0", + "libsodium-wrappers": "^0.8.2", + "mkdirp": "^3.0.1", + "msgpackr": "^1.11.2" + } + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index aa3b8af..79b8f38 100644 --- a/package.json +++ b/package.json @@ -7,12 +7,13 @@ "mounttest": "export $(cat .env | xargs) && RUST_LOG=none,fuse=debug,sqlitesuperfs=info cargo run -- test1 $(pwd)/testdir", "mounttestq": "export $(cat .env | xargs) && RUST_LOG=none,sqlitesuperfs=info cargo run -- test1 $(pwd)/testdir", "mounttest0": "export $(cat .env | xargs) && RUST_LOG=none,fuse=debug,sqlitesuperfs=info cargo run -- --ttl 0 test1 $(pwd)/testdir", - "test": "tape test/1basic.js && tape test/2dir.js && tape test/3rw.js && tape test/4link.js && tape test/5misc.js && tape test/6sql.js", - "test-umount": "tape test/7umount.js" + "test": "tape test/1basic.js && tape test/2dir.js && tape test/3rw.js && tape test/4link.js && tape test/5misc.js && tape test/6sql.js && tape test/7raft.js", + "test-umount": "tape test/8umount.js" }, "devDependencies": { "tape": "^5.9.0", "better-sqlite3": "^12.4.6", + "tinyraftplus": "^0.3.0", "split": "^1.0.1" } } diff --git a/test/7raft.js b/test/7raft.js new file mode 100644 index 0000000..708ae99 --- /dev/null +++ b/test/7raft.js @@ -0,0 +1,82 @@ +const fs = require('fs') +const test = require('tape') +const Database = require('better-sqlite3') +const { FsLog } = require('tinyraftplus') + +const DIR = `./testdir` +const rmr = (path) => fs.rmSync(path, { recursive: true }) +const empty = (dir) => { + const arr = fs.readdirSync(dir, { withFileTypes: true }) + arr.forEach((entry) => { + const path = `${entry.path}/${entry.name}` + rmr(path) + }) +} + +const toBuf = (obj) => { + if (obj === null) { return null } + obj = JSON.stringify(obj) + return Buffer.from(obj, 'utf8') +} + +const toObj = (buf) => { + if (buf === null) { return null } + return JSON.parse(buf.toString('utf8')) +} + +test('test append, open, close, new', async (t) => { + empty(DIR) + let log = new FsLog(`${DIR}/`, 'test') + t.teardown(() => log.close()) + + await log.del() + await log.open() + t.equal(log.seq, -1n, 'seq = -1') + t.equal(log.head, null, 'head = null') + + // open, close same + let data = { a: 1 } + let seq = await log.append(toBuf(data)) + t.equal(seq, 0n, 'seq = 0') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + data = { bb: 2 } + seq = await log.append(toBuf(data)) + t.equal(seq, 1n, 'seq = 1') + t.equal(log.seq, 1n, 'seq = 1') + t.deepEqual(toObj(log.head), data, 'head = data') + + data = { ccc: 3 } + seq = await log.append(toBuf(data)) + t.equal(seq, 2n, 'seq = 2') + t.equal(log.seq, 2n, 'seq = 2') + t.deepEqual(toObj(log.head), data, 'head = data') + await log.close() + + // open, close same + await log.open() + t.equal(log.seq, 2n, 'seq = 2 again') + t.deepEqual(toObj(log.head), data, 'head = data again') + + data = { d: 4 } + seq = await log.append(toBuf(data)) + t.equal(seq, 3n, 'seq = 3') + t.equal(log.seq, 3n, 'seq = 3') + t.deepEqual(toObj(log.head), data, 'head = data') + await log.close() + + // new + log = new FsLog(`${DIR}/`, 'test') + await log.open() + t.equal(log.seq, 3n, 'seq = 3 again') + t.deepEqual(toObj(log.head), data, 'head = data again') + + data = { ee: 5 } + seq = await log.append(toBuf(data)) + t.equal(seq, 4n, 'seq = 4') + t.equal(log.seq, 4n, 'seq = 4') + t.deepEqual(toObj(log.head), data, 'head = data') + + t.end() +}) diff --git a/test/7umount.js b/test/8umount.js similarity index 100% rename from test/7umount.js rename to test/8umount.js From 94b5793a24ecde662d148e6f4572793992d0c3dd Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 11:29:30 -0500 Subject: [PATCH 4/8] more raft tests --- test/7raft.js | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/test/7raft.js b/test/7raft.js index 708ae99..6d8f7f6 100644 --- a/test/7raft.js +++ b/test/7raft.js @@ -24,7 +24,7 @@ const toObj = (buf) => { return JSON.parse(buf.toString('utf8')) } -test('test append, open, close, new', async (t) => { +test('raft append, open, close, new', async (t) => { empty(DIR) let log = new FsLog(`${DIR}/`, 'test') t.teardown(() => log.close()) @@ -77,6 +77,96 @@ test('test append, open, close, new', async (t) => { t.equal(seq, 4n, 'seq = 4') t.equal(log.seq, 4n, 'seq = 4') t.deepEqual(toObj(log.head), data, 'head = data') + t.end() +}) + +test('raft append one, close, open, append', async (t) => { + empty(DIR) + const log = new FsLog(`${DIR}/`, 'test') + t.teardown(() => log.close()) + + await log.del() + await log.open() + + t.equal(log.seq, -1n, 'seq = -1') + t.equal(log.head, null, 'head = null') + + let data = { a: 1 } + let seq = await log.append(toBuf(data)) + t.equal(seq, 0n, 'seq = 0') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + await log.close() + + await log.open() + t.equal(log.seq, 0n, 'seq = 0 again') + t.deepEqual(toObj(log.head), data, 'head = data again') + + data = { b: 2 } + seq = await log.append(toBuf(data)) + t.equal(seq, 1n, 'seq = 1') + t.equal(log.seq, 1n, 'seq = 1') + t.deepEqual(toObj(log.head), data, 'head = data') + t.end() +}) +test('raft rollback first', async (t) => { + const rollbackCb = (seq) => { + if (seq === 0n) { throw new Error('test roll') } + } + + const opts = { rollbackCb } + const log = new FsLog(`${DIR}/`, 'test', opts) + t.teardown(() => log.close()) + + await log.del() + await log.open() + + t.equal(log.seq, -1n, 'seq = -1') + t.equal(log.head, null, 'head = null') + + try { + const data = { a: 1 } + await log.append(toBuf(data)) + t.fail('no error thrown') + } catch (err) { + t.ok(err.message.includes('test roll'), 'error thrown') + } + + t.equal(log.seq, -1n, 'seq = -1 again') + t.equal(log.head, null, 'head = null again') + t.end() +}) + +test('raft rollback second', async (t) => { + const rollbackCb = (seq) => { + if (seq === 1n) { throw new Error('test roll') } + } + + const opts = { rollbackCb } + const log = new FsLog(`${DIR}/`, 'test', opts) + t.teardown(() => log.close()) + + await log.del() + await log.open() + + const data = { a: 1 } + const seq = await log.append(toBuf(data)) + t.equal(seq, 0n, 'seq = 0') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + try { + await log.append(toBuf({ b: 2 })) + t.fail('no error thrown') + } catch (err) { + t.ok(err.message.includes('test roll'), 'error thrown') + } + + await log.close() + await log.open() + t.pass('restart ok') + t.equal(log.seq, 0n, 'seq = 0 again') + t.deepEqual(toObj(log.head), data, 'head = data again') t.end() }) From eec77fdadd360b00d1a8142249b3e6ed8cee8952 Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 11:47:17 -0500 Subject: [PATCH 5/8] more raft tests --- test/7raft.js | 39 ++++++++++++++++++++ test/8umount.js | 97 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/test/7raft.js b/test/7raft.js index 6d8f7f6..30afc89 100644 --- a/test/7raft.js +++ b/test/7raft.js @@ -111,6 +111,7 @@ test('raft append one, close, open, append', async (t) => { }) test('raft rollback first', async (t) => { + empty(DIR) const rollbackCb = (seq) => { if (seq === 0n) { throw new Error('test roll') } } @@ -139,6 +140,7 @@ test('raft rollback first', async (t) => { }) test('raft rollback second', async (t) => { + empty(DIR) const rollbackCb = (seq) => { if (seq === 1n) { throw new Error('test roll') } } @@ -170,3 +172,40 @@ test('raft rollback second', async (t) => { t.deepEqual(toObj(log.head), data, 'head = data again') t.end() }) + +test('raft txn', async (t) => { + empty(DIR) + const log = new FsLog(`${DIR}/`, 'test') + t.teardown(() => log.close()) + + await log.del() + await log.open() + + t.equal(log.seq, -1n, 'seq = -1') + t.equal(log.head, null, 'head = null') + + let data = { a: 1 } + let txn = await log.txn() + let seq = await txn.append(toBuf(data)) + t.equal(seq, 0n, 'seq = 0') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + await txn.commit() + t.pass('commit ok') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + data = { b: 2 } + txn = await log.txn() + seq = await txn.append(toBuf(data)) + t.equal(seq, 1n, 'seq = 1') + t.equal(log.seq, 1n, 'seq = 1') + t.deepEqual(toObj(log.head), data, 'head = data') + + await txn.abort() + t.pass('abort ok') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), { a: 1 }, 'head = data') + t.end() +}) diff --git a/test/8umount.js b/test/8umount.js index c1d87b4..dc324af 100644 --- a/test/8umount.js +++ b/test/8umount.js @@ -1,6 +1,7 @@ const test = require('tape') const split = require('split') const Database = require('better-sqlite3') +const { FsLog } = require('tinyraftplus') const exec = require('child_process').exec const spawn = require('child_process').spawn @@ -204,6 +205,102 @@ test('update TRUNCATE SIGINT', (t) => testUpdate(t, 'TRUNCATE', 'SIGINT')) test('update DELETE SIGTERM', (t) => testUpdate(t, 'DELETE', 'SIGTERM')) test('update TRUNCATE SIGTERM', (t) => testUpdate(t, 'TRUNCATE', 'SIGTERM')) +const toBuf = (obj) => { + if (obj === null) { return null } + obj = JSON.stringify(obj) + return Buffer.from(obj, 'utf8') +} + +const toObj = (buf) => { + if (buf === null) { return null } + return JSON.parse(buf.toString('utf8')) +} + +async function testRaft(t, sig) { + let ended = false + const errCb = (err) => { + if (ended) { return } + t.fail(err.message) + } + + await reset() + t.pass('reset ok') + + let child = await mount(errCb) + t.pass('mount ok') + + const kill = () => { + ended = true + child.once('exit', (code) => console.log('ssfs >> exit', code)) + child.kill(sig) + return sleep(500) + } + + t.teardown(kill) + + // open, close same + let log = new FsLog(`${DIR}/`, 'raft') + await log.open() + t.pass('open ok') + t.equal(log.seq, -1n, 'seq = -1') + t.equal(log.head, null, 'head = null') + + let data = { a: 1 } + let seq = await log.append(toBuf(data)) + t.equal(seq, 0n, 'seq = 0') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + data = { bb: 2 } + seq = await log.append(toBuf(data)) + t.equal(seq, 1n, 'seq = 1') + t.equal(log.seq, 1n, 'seq = 1') + t.deepEqual(toObj(log.head), data, 'head = data') + + data = { ccc: 3 } + seq = await log.append(toBuf(data)) + t.equal(seq, 2n, 'seq = 2') + t.equal(log.seq, 2n, 'seq = 2') + t.deepEqual(toObj(log.head), data, 'head = data') + await log.close() + + // open, close same + await log.open() + t.equal(log.seq, 2n, 'seq = 2 again') + t.deepEqual(toObj(log.head), data, 'head = data again') + + data = { d: 4 } + seq = await log.append(toBuf(data)) + t.equal(seq, 3n, 'seq = 3') + t.equal(log.seq, 3n, 'seq = 3') + t.deepEqual(toObj(log.head), data, 'head = data') + + await kill() + await umount() + ended = false + child = await mount(errCb) + t.pass('mount again ok') + + log = new FsLog(`${DIR}/`, 'raft') + await log.open() + t.pass('open again ok') + t.equal(log.seq, 3n, 'seq = 3 again') + t.deepEqual(toObj(log.head), data, 'head = data again') + + data = { ee: 5 } + seq = await log.append(toBuf(data)) + t.equal(seq, 4n, 'seq = 4') + t.equal(log.seq, 4n, 'seq = 4') + t.deepEqual(toObj(log.head), data, 'head = data') + + await log.close() + t.pass('close ok') + t.end() +} + +test('raft SIGINT', (t) => testRaft(t, 'SIGINT')) +test('raft SIGTERM', (t) => testRaft(t, 'SIGTERM')) + test('reset', async (t) => { await reset() t.pass('reset ok') From f7fb10e8b0a7f3fb750e79f04942c4dcb4c26ca3 Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 12:10:29 -0500 Subject: [PATCH 6/8] tests pass --- test/8umount.js | 88 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/test/8umount.js b/test/8umount.js index dc324af..c466bd0 100644 --- a/test/8umount.js +++ b/test/8umount.js @@ -216,6 +216,8 @@ const toObj = (buf) => { return JSON.parse(buf.toString('utf8')) } +const noop = () => {} + async function testRaft(t, sig) { let ended = false const errCb = (err) => { @@ -264,7 +266,7 @@ async function testRaft(t, sig) { t.deepEqual(toObj(log.head), data, 'head = data') await log.close() - // open, close same + // open same await log.open() t.equal(log.seq, 2n, 'seq = 2 again') t.deepEqual(toObj(log.head), data, 'head = data again') @@ -281,6 +283,7 @@ async function testRaft(t, sig) { child = await mount(errCb) t.pass('mount again ok') + // new log = new FsLog(`${DIR}/`, 'raft') await log.open() t.pass('open again ok') @@ -301,6 +304,89 @@ async function testRaft(t, sig) { test('raft SIGINT', (t) => testRaft(t, 'SIGINT')) test('raft SIGTERM', (t) => testRaft(t, 'SIGTERM')) +async function testRaftTxn(t, sig) { + let ended = false + const errCb = (err) => { + if (ended) { return } + t.fail(err.message) + } + + await reset() + t.pass('reset ok') + + let child = await mount(errCb) + t.pass('mount ok') + + const kill = () => { + ended = true + child.once('exit', (code) => console.log('ssfs >> exit', code)) + child.kill(sig) + return sleep(500) + } + + t.teardown(kill) + + let log = new FsLog(`${DIR}/`, 'raft') + await log.open() + t.pass('open ok') + t.equal(log.seq, -1n, 'seq = -1') + t.equal(log.head, null, 'head = null') + + // commit + let data = { a: 1 } + let txn = await log.txn() + let seq = await txn.append(toBuf(data)) + t.equal(seq, 0n, 'seq = 0') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + await txn.commit() + t.pass('commit ok') + t.equal(log.seq, 0n, 'seq = 0') + t.deepEqual(toObj(log.head), data, 'head = data') + + // rolled back + data = { b: 2 } + txn = await log.txn() + seq = await txn.append(toBuf(data)) + t.equal(seq, 1n, 'seq = 1') + t.equal(log.seq, 1n, 'seq = 1') + t.deepEqual(toObj(log.head), data, 'head = data') + + await kill() + await umount() + ended = false + child = await mount(errCb) + t.pass('mount again ok') + + // rolled back + log = new FsLog(`${DIR}/`, 'raft') + await log.open() + t.pass('open again ok') + t.equal(log.seq, 0n, 'seq = 0 again') + t.deepEqual(toObj(log.head), { a: 1 }, 'head = data') + + await log.close() + t.pass('close ok') + await log.open() + t.pass('open again again ok') + t.equal(log.seq, 0n, 'seq = 0 again') + t.deepEqual(toObj(log.head), { a: 1 }, 'head = data') + + await log.close() + t.pass('close again ok') + t.end() +} + +test('raft SIGINT txn', (t) => testRaftTxn(t, 'SIGINT')) +test('raft SIGTERM txn', (t) => testRaftTxn(t, 'SIGTERM')) + +process.on('uncaughtException', (err) => { + if (err.message && err.message.includes('Closing file descriptor')) { return } + console.log('Uncaught exception:', err) + process.exit(1) +}) + test('reset', async (t) => { await reset() t.pass('reset ok') From c572b678e5bfa00f109cd76b030602e9641d5265 Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 12:34:19 -0500 Subject: [PATCH 7/8] update node version used for ci --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 7ed7cf2..c839550 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -30,7 +30,7 @@ jobs: - name: Setup node uses: actions/setup-node@v4 with: - node-version: '20.11.0' + node-version: '23.7.0' cache: 'npm' - name: Build rust From 670d5fd78bae4deb1d5d0660429c135c564213b8 Mon Sep 17 00:00:00 2001 From: rhodey Date: Fri, 20 Feb 2026 12:37:55 -0500 Subject: [PATCH 8/8] CI remove log level info --- .github/workflows/run.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index c839550..ec7635b 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -41,7 +41,7 @@ jobs: mkdir -p ./testdir export psql_url=postgresql://psql:psql@localhost:5432/psql export encryption_pass=supersuper - RUST_LOG=info cargo run -- -o 'allow_other' --ttl 0 test1 $(pwd)/testdir & + cargo run -- -o 'allow_other' --ttl 0 test1 $(pwd)/testdir & sleep 5 npm install npm run test @@ -51,7 +51,7 @@ jobs: run: | export psql_url=postgresql://psql:psql@localhost:5432/psql export encryption_pass=supersuper - RUST_LOG=info cargo run -- -o 'allow_other' --ttl 180 test1 $(pwd)/testdir & + cargo run -- -o 'allow_other' --ttl 180 test1 $(pwd)/testdir & sleep 5 npm run test umount $(pwd)/testdir