Merge "SQLite master branch - Dec 15th 2022" from Piotr Sarna

Author: penberg

ext/misc/base64.c

@@ -91,7 +91,8 @@ static const ubyte b64DigitValues[128] = {
 static const char b64Numerals[64+1]
 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
-#define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80)
+#define BX_DV_PROTO(c) \
+  ((((ubyte)(c))<0x80)? (ubyte)(b64DigitValues[(ubyte)(c)]) : 0x80)
 #define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80)
 #define IS_BX_WS(bdp) ((bdp)==WS)
 #define IS_BX_PAD(bdp) ((bdp)==PC)

ext/misc/basexx.c

@@ -69,7 +69,6 @@ int sqlite3_basexx_init(sqlite3 *db, char **pzErr,
   init_api_ptr(pApi);
   int rc1 = BASE64_INIT(db);
   int rc2 = BASE85_INIT(db);
-  int rc = SQLITE_OK;
 
   if( rc1==SQLITE_OK && rc2==SQLITE_OK ){
     BASE64_EXPOSE(db, pzErr);

ext/rbu/rbuB.test

@@ -48,7 +48,7 @@ do_test 1.2 {
 
 do_test 1.3 {
   set ::errlog
-} {SQLITE_NOTICE_RECOVER_WAL SQLITE_INTERNAL}
+} {SQLITE_NOTICE_RECOVER_WAL SQLITE_NOTICE_RBU}
 
 do_execsql_test 1.4 {
   SELECT * FROM t1

ext/rbu/sqlite3rbu.c

@@ -3031,11 +3031,11 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
   **     no-ops. These locks will not be released until the connection
   **     is closed.
   **
-  **   * Attempting to xSync() the database file causes an SQLITE_INTERNAL 
+  **   * Attempting to xSync() the database file causes an SQLITE_NOTICE 
   **     error.
   **
   ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the
-  ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[]
+  ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[]
   ** array populated with a set of (frame -> page) mappings. Because the 
   ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy 
   ** data from the wal file into the database file according to the 
@@ -3045,7 +3045,7 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
     int rc2;
     p->eStage = RBU_STAGE_CAPTURE;
     rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
-    if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
+    if( rc2!=SQLITE_NOTICE ) p->rc = rc2;
   }
 
   if( p->rc==SQLITE_OK && p->nFrame>0 ){
@@ -3091,7 +3091,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){
 
   if( pRbu->mLock!=mReq ){
     pRbu->rc = SQLITE_BUSY;
-    return SQLITE_INTERNAL;
+    return SQLITE_NOTICE_RBU;
   }
 
   pRbu->pgsz = iAmt;
@@ -4478,7 +4478,7 @@ void sqlite3rbu_rename_handler(
 **     database file are recorded. xShmLock() calls to unlock the same
 **     locks are no-ops (so that once obtained, these locks are never
 **     relinquished). Finally, calls to xSync() on the target database
-**     file fail with SQLITE_INTERNAL errors.
+**     file fail with SQLITE_NOTICE errors.
 */
 
 static void rbuUnlockShm(rbu_file *p){
@@ -4757,7 +4757,7 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){
   rbu_file *p = (rbu_file *)pFile;
   if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){
     if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
-      return SQLITE_INTERNAL;
+      return SQLITE_NOTICE_RBU;
     }
     return SQLITE_OK;
   }

ext/session/sessionat.test

@@ -243,6 +243,60 @@ eval [string map [list %WR% $trailing] {
     sqlite3changeset_apply db $cinv xConflict
     execsql { SELECT * FROM t7 }
   } {1 1 ccc 2 2 ccc 3 3 ccc}
+
+  #-----------------------------------------------------------------------
+  reset_test
+  do_execsql_test $tn.7.0 {
+    CREATE TABLE t8(a PRIMARY KEY, b, c);
+  }
+  do_execsql_test -db db2 $tn.7.1 {
+    CREATE TABLE t8(a PRIMARY KEY, b, c, d DEFAULT 'D', e DEFAULT 'E');
+  }
+
+  do_then_apply_sql {
+    INSERT INTO t8 VALUES(1, 2, 3);
+    INSERT INTO t8 VALUES(4, 5, 6);
+  }
+  do_execsql_test $tn.7.2.1 {
+    SELECT * FROM t8
+  } {1 2 3    4 5 6}
+  do_execsql_test -db db2 $tn.7.2.2 {
+    SELECT * FROM t8
+  } {1 2 3 D E   4 5 6 D E}
+
+  do_then_apply_sql {
+    UPDATE t8 SET c=45 WHERE a=4;
+  }
+  do_execsql_test $tn.7.3.1 {
+    SELECT * FROM t8
+  } {1 2 3    4 5 45}
+  do_execsql_test -db db2 $tn.7.3.2 {
+    SELECT * FROM t8
+  } {1 2 3 D E   4 5 45 D E}
+
+  #-----------------------------------------------------------------------
+  reset_test
+  do_execsql_test $tn.8.0 {
+    CREATE TABLE t9(a PRIMARY KEY, b, c, d, e, f, g, h);
+  }
+  do_execsql_test -db db2 $tn.8.1 {
+    CREATE TABLE t9(a PRIMARY KEY, b, c, d, e, f, g, h, i, j, k, l);
+  }
+  do_then_apply_sql {
+    INSERT INTO t9 VALUES(1, 2, 3, 4, 5, 6, 7, 8);
+  }
+  do_then_apply_sql {
+    UPDATE t9 SET h=450 WHERE a=1
+  }
+  do_execsql_test -db db2 $tn.8.2 {
+    SELECT * FROM t9
+  } {1 2 3 4 5 6 7 450 {} {} {} {}}
+  do_then_apply_sql {
+    UPDATE t9 SET h=NULL
+  }
+  do_execsql_test -db db2 $tn.8.2 {
+    SELECT * FROM t9
+  } {1 2 3 4 5 6 7 {} {} {} {} {}}
 }]
 }
 

ext/session/sqlite3session.c

@@ -3327,6 +3327,22 @@ static int sessionChangesetNextOne(
       if( p->op==SQLITE_INSERT ) p->op = SQLITE_DELETE;
       else if( p->op==SQLITE_DELETE ) p->op = SQLITE_INSERT;
     }
+
+    /* If this is an UPDATE that is part of a changeset, then check that
+    ** there are no fields in the old.* record that are not (a) PK fields,
+    ** or (b) also present in the new.* record. 
+    **
+    ** Such records are technically corrupt, but the rebaser was at one
+    ** point generating them. Under most circumstances this is benign, but
+    ** can cause spurious SQLITE_RANGE errors when applying the changeset. */
+    if( p->bPatchset==0 && p->op==SQLITE_UPDATE){
+      for(i=0; i<p->nCol; i++){
+        if( p->abPK[i]==0 && p->apValue[i+p->nCol]==0 ){
+          sqlite3ValueFree(p->apValue[i]);
+          p->apValue[i] = 0;
+        }
+      }
+    }
   }
 
   return SQLITE_ROW;
@@ -5523,7 +5539,7 @@ static void sessionAppendPartialUpdate(
         if( !pIter->abPK[i] && a1[0] ) bData = 1;
         memcpy(pOut, a1, n1);
         pOut += n1;
-      }else if( a2[0]!=0xFF ){
+      }else if( a2[0]!=0xFF && a1[0] ){
         bData = 1;
         memcpy(pOut, a2, n2);
         pOut += n2;

ext/session/test_session.c

@@ -98,6 +98,19 @@ int sql_exec_changeset(
 }
 /************************************************************************/
 
+
+#ifdef SQLITE_DEBUG
+static int sqlite3_test_changeset(int, void *, char **);
+static void assert_changeset_is_ok(int n, void *p){
+  int rc = 0;
+  char *z = 0;
+  rc = sqlite3_test_changeset(n, p, &z);
+  assert( z==0 );
+}
+#else
+# define assert_changeset_is_ok(n,p)
+#endif
+
 /*
 ** Tclcmd: sql_exec_changeset DB SQL
 */
@@ -127,6 +140,7 @@ static int SQLITE_TCLAPI test_sql_exec_changeset(
     return TCL_ERROR;
   }
 
+  assert_changeset_is_ok(nChangeset, pChangeset);
   Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pChangeset, nChangeset));
   sqlite3_free(pChangeset);
   return TCL_OK;
@@ -295,6 +309,7 @@ static int SQLITE_TCLAPI test_session_cmd(
         }
       }
       if( rc==SQLITE_OK ){
+        assert_changeset_is_ok(o.n, o.p);
         Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(o.p, o.n)); 
       }
       sqlite3_free(o.p);
@@ -953,6 +968,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_invert(
   if( rc!=SQLITE_OK ){
     rc = test_session_error(interp, rc, 0);
   }else{
+    assert_changeset_is_ok(sOut.n, sOut.p);
     Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n));
   }
   sqlite3_free(sOut.p);
@@ -1001,6 +1017,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat(
   if( rc!=SQLITE_OK ){
     rc = test_session_error(interp, rc, 0);
   }else{
+    assert_changeset_is_ok(sOut.n, sOut.p);
     Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n));
   }
   sqlite3_free(sOut.p);
@@ -1236,6 +1253,7 @@ static int SQLITE_TCLAPI test_rebaser_cmd(
       }
 
       if( rc==SQLITE_OK ){
+        assert_changeset_is_ok(sOut.n, sOut.p);
         Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(sOut.p, sOut.n));
       }
       sqlite3_free(sOut.p);
@@ -1282,6 +1300,107 @@ static int SQLITE_TCLAPI test_sqlite3rebaser_create(
   return TCL_OK;
 }
 
+/*
+** Run some sanity checks on the changeset in nChangeset byte buffer
+** pChangeset. If any fail, return a non-zero value and, optionally,
+** set output variable (*pzErr) to point to a buffer containing an
+** English language error message describing the problem. In this
+** case it is the responsibility of the caller to free the buffer
+** using sqlite3_free().
+**
+** Or, if the changeset appears to be well-formed, this function
+** returns SQLITE_OK and sets (*pzErr) to NULL.
+*/
+static int sqlite3_test_changeset(
+  int nChangeset,
+  void *pChangeset,
+  char **pzErr
+){
+  sqlite3_changeset_iter *pIter = 0;
+  char *zErr = 0;
+  int rc = SQLITE_OK;
+  int bPatch = (nChangeset>0 && ((char*)pChangeset)[0]=='P');
+
+  rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter) ){
+      unsigned char *aPk = 0;
+      int nCol = 0;
+      int op = 0;
+      const char *zTab = 0;
+
+      sqlite3changeset_pk(pIter, &aPk, &nCol);
+      sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0);
+
+      if( op==SQLITE_UPDATE ){
+        int iCol;
+        for(iCol=0; iCol<nCol; iCol++){
+          sqlite3_value *pNew = 0;
+          sqlite3_value *pOld = 0;
+          sqlite3changeset_new(pIter, iCol, &pNew);
+          sqlite3changeset_old(pIter, iCol, &pOld);
+
+          if( aPk[iCol] ){
+            if( pOld==0 ) rc = SQLITE_ERROR;
+          }else if( bPatch ){
+            if( pOld ) rc = SQLITE_ERROR;
+          }else{
+            if( (pOld==0)!=(pNew==0) ) rc = SQLITE_ERROR;
+          }
+
+          if( rc!=SQLITE_OK ){
+            zErr = sqlite3_mprintf(
+                "unexpected SQLITE_UPDATE (bPatch=%d pk=%d pOld=%d pNew=%d)",
+                bPatch, (int)aPk[iCol], pOld!=0, pNew!=0
+            );
+            break;
+          }
+        }
+      }
+    }
+    rc2 = sqlite3changeset_finalize(pIter);
+    if( rc==SQLITE_OK ){
+      rc = rc2;
+    }
+  }
+
+  *pzErr = zErr;
+  return rc;
+}
+
+/*
+** test_changeset CHANGESET
+*/
+static int SQLITE_TCLAPI test_changeset(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  void *pChangeset = 0;           /* Buffer containing changeset */
+  int nChangeset = 0;             /* Size of buffer aChangeset in bytes */
+  int rc = SQLITE_OK;
+  char *z = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CHANGESET");
+    return TCL_ERROR;
+  }
+  pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[1], &nChangeset);
+
+  Tcl_ResetResult(interp);
+  rc = sqlite3_test_changeset(nChangeset, pChangeset, &z);
+  if( rc!=SQLITE_OK ){
+    char *zErr = sqlite3_mprintf("(%d) - \"%s\"", rc, z);
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
+    sqlite3_free(zErr);
+  }
+  sqlite3_free(z);
+
+  return rc ? TCL_ERROR : TCL_OK;
+}
+
 /*
 ** tclcmd: sqlite3rebaser_configure OP VALUE
 */
@@ -1337,6 +1456,7 @@ int TestSession_Init(Tcl_Interp *interp){
     { "sql_exec_changeset", test_sql_exec_changeset },
     { "sqlite3rebaser_create", test_sqlite3rebaser_create },
     { "sqlite3session_config", test_sqlite3session_config },
+    { "test_changeset", test_changeset },
   };
   int i;