Add another cache

.github/workflows/run.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main
   pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+    types: [opened, synchronize, reopened]
     branches:
       - main
 

README.md

@@ -55,7 +55,7 @@ SQLitesuperfs passes `make test` in the official SQLite source tree. So you can:
 + Run `make test` from the mount
 
 ## Performance
-When the PSQL server is localhost SQLite is 15% faster than native FS thanks to SQLitesuperfs buffering. When the PSQL server has 0.2ms simulated RTT latency SQLite is 2x to 3x slower than native FS. SQLitesuperfs is primarily about *private hosted SQLite impossible => possible*. SQLitesuperfs even at 100x would be an achievement compared to Fully Homomorphic Encryption being 100,000x to 1,000,000x.
+When the PSQL server is localhost SQLite is 15% faster than native FS thanks to SQLitesuperfs buffering. When the PSQL server has 0.2ms simulated RTT latency SQLite is 2x to 3x slower than native FS. SQLitesuperfs is primarily about *private hosted SQLite impossible => possible*. SQLitesuperfs even at 100x would be an achievement compared to Fully Homomorphic Encryption being 100,000x to 1,000,000x. Also remember SQLite on native FS is fast!
 
 The read thread pool described earlier will help and additionally a custom [VFS](https://sqlite.org/vfs.html) can be added to improve writes but TBH I am very happy already to build apps on this so I dont say I will be back here to add either *super* soon.
 

src/fs/mod.rs

@@ -3,6 +3,7 @@ use log::debug;
 use libc::c_int;
 use std::path::Path;
 use std::collections::HashMap;
+use std::collections::HashSet;
 
 use std::ffi::{OsStr, OsString};
 use std::os::unix::ffi::OsStrExt;
@@ -147,6 +148,7 @@ type Cache = HashMap<u64, Inode>;
 type Blocks = HashMap<u64, Block>;
 type BlockCache = HashMap<u64, Blocks>;
 type Names = Vec<Vec<String>>;
+type Missing = HashSet<(u64, String)>;
 
 #[allow(dead_code)]
 pub struct Fs {
@@ -162,6 +164,7 @@ pub struct Fs {
   names: Names,
   buffers: BlockCache,
   diffo: Blocks,
+  missing: Missing,
   pgdb: PgDb,
 }
 
@@ -174,11 +177,12 @@ impl Fs {
     let inodes: Cache = HashMap::new();
     let buffers: BlockCache = HashMap::new();
     let diffo: Blocks = HashMap::new();
+    let missing: Missing = HashSet::new();
     Fs {
       uid, gid,
       block_sz, buffer_bytes, ttl_ms,
       fh_map, fh_dir_map, fh_next: 0,
-      inodes, names, buffers, diffo, pgdb,
+      inodes, names, buffers, diffo, missing, pgdb,
     }
   }
 
@@ -397,9 +401,17 @@ impl Filesystem for Fs {
 
   fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) {
     let name = b64_encode(name);
+    // cache for a common code path
+    if self.missing.contains(&(parent, name.clone())) {
+      reply.error(libc::ENOENT);
+      return
+    }
     let inode = match self.pgdb.lookup(parent, &name) {
       Ok(Some(inode)) => inode,
-      Ok(None) => return reply.error(libc::ENOENT),
+      Ok(None) => {
+        self.missing.insert((parent, name));
+        return reply.error(libc::ENOENT);
+      },
       Err(errno) => return reply.error(errno),
     };
     let ttl = to_ts(self.ttl_ms);
@@ -502,6 +514,7 @@ impl Filesystem for Fs {
       Err(errno) => return reply.error(errno),
     };
     self.inodes.insert(inode.id, inode.clone());
+    self.missing.remove(&(parent, name));
     let ttl = to_ts(self.ttl_ms);
     let gen = 0;
     reply.entry(&ttl, &inode.attr(self.block_sz), gen);
@@ -550,6 +563,7 @@ impl Filesystem for Fs {
     };
     inode.open += 1;
     self.inodes.insert(inode.id, inode.clone());
+    self.missing.remove(&(parent, name));
     self.fh_next += 1;
     let fh = self.fh_next;
     self.fh_map.insert(fh, inode.clone());
@@ -932,6 +946,7 @@ impl Filesystem for Fs {
       Ok(inode) => inode,
       Err(errno) => return reply.error(errno),
     };
+    self.missing.remove(&(parent, name));
     let mut parent = match self.get_ino(parent) {
       Ok(Some(inode)) => inode,
       Ok(None) => return reply.error(libc::EIO),
@@ -988,6 +1003,7 @@ impl Filesystem for Fs {
     };
     inode.open = orig.open;
     self.inodes.insert(ino, inode.clone());
+    self.missing.remove(&(new_parent, new_name));
     let ttl = to_ts(self.ttl_ms);
     let gen = 0;
     reply.entry(&ttl, &inode.attr(self.block_sz), gen);
@@ -1003,6 +1019,7 @@ impl Filesystem for Fs {
       Err(errno) => return reply.error(errno),
     };
     self.inodes.insert(inode.id, inode.clone());
+    self.missing.clear();
     let ttl = to_ts(self.ttl_ms);
     let gen = 0;
     reply.entry(&ttl, &inode.attr(self.block_sz), gen);
@@ -1056,6 +1073,7 @@ impl Filesystem for Fs {
     self.inodes.insert(inode.id, inode);
     self.inodes.insert(p1.id, p1);
     self.inodes.insert(p2.id, p2);
+    self.missing.remove(&(new_parent, new_name));
     reply.ok();
   }
 

test/6sql.js

@@ -152,7 +152,7 @@ test('sql big txn', (t) => {
   t.end()
 })
 
-test('sql two txn', (t) => {
+test('sql 2 txn', (t) => {
   empty(DIR)
 
   const file = `${DIR}/test.db`
@@ -188,7 +188,7 @@ test('sql two txn', (t) => {
   t.end()
 })
 
-test('sql many txn', (t) => {
+test('sql 10 txn', (t) => {
   empty(DIR)
 
   const file = `${DIR}/test.db`
@@ -214,6 +214,7 @@ test('sql many txn', (t) => {
   })
 
   let ms = 0
+  let avg = 0
   for (let i = 0; i < 10; i++) {
     db.prepare('delete from users').run()
     begin = Date.now()
@@ -222,10 +223,14 @@ test('sql many txn', (t) => {
       insertMany(two)
     }
     ms = Date.now() - begin
+    avg += ms
     t.pass(`insert ok ${i}`)
     t.pass(`insert ${i} ${ms}ms`)
   }
 
+  avg = (avg / 10).toFixed(0)
+  t.pass(`insert avg ${avg}ms`)
+
   const row = db.prepare('select count(id) as c from users').get()
   t.equal(row.c, users.length, 'count ok')