From 591fe5ff3f15279b6be0034997f9492837734f22 Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Wed, 13 May 2026 09:12:00 +0100 Subject: [PATCH] cgosqlite: update to SQLite 3.53.1 ```console $ ./update-sqlite.sh 'https://sqlite.org/2026/sqlite-amalgamation-3530100.zip' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 2869k 100 2869k 0 0 869k 0 0:00:03 0:00:03 --:--:-- 869k Archive: sqlite-amalgamation-3530100.zip creating: sqlite-amalgamation-3530100 inflating: sqlite-amalgamation-3530100/sqlite3.c inflating: sqlite-amalgamation-3530100/shell.c inflating: sqlite-amalgamation-3530100/sqlite3.h inflating: sqlite-amalgamation-3530100/sqlite3ext.h Next step: update the expected version numbers in release_test.go ``` Updates tailscale/corp#41750 Signed-off-by: Alex Chan --- cgosqlite/release_test.go | 4 +- cgosqlite/shell.c.disabled | 7290 ++++++++++++++++++------------------ cgosqlite/sqlite3.c | 3317 +++++++++++----- cgosqlite/sqlite3.h | 364 +- update-sqlite.sh | 2 + 5 files changed, 6394 insertions(+), 4583 deletions(-) diff --git a/cgosqlite/release_test.go b/cgosqlite/release_test.go index bbb0003..5f65b6a 100644 --- a/cgosqlite/release_test.go +++ b/cgosqlite/release_test.go @@ -5,7 +5,7 @@ package cgosqlite import "testing" func TestSQLiteVersion(t *testing.T) { - const want = "3.52.0" + const want = "3.53.1" got := SQLiteVersion() if got != want { t.Fatalf("wrong version, want %s, got %s", want, got) @@ -13,7 +13,7 @@ func TestSQLiteVersion(t *testing.T) { } func TestSQLiteVersionNumber(t *testing.T) { - const want = 3052000 + const want = 3053001 got := SQLiteVersionNumber() if got != want { t.Fatalf("wrong version, want %d, got %d", want, got) diff --git a/cgosqlite/shell.c.disabled b/cgosqlite/shell.c.disabled index fd244bf..57faceb 100644 --- a/cgosqlite/shell.c.disabled +++ b/cgosqlite/shell.c.disabled @@ -61,6 +61,22 @@ typedef unsigned int u32; typedef unsigned short int u16; +/* +** Limit input nesting via .read or any other input redirect. +** It's not too expensive, so a generous allowance can be made. +*/ +#define MAX_INPUT_NESTING 25 + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + /* ** Optionally #include a user-defined header, whereby compilation options ** may be set prior to where they take effect, but after platform setup. @@ -138,6 +154,7 @@ typedef unsigned short int u16; #include #include #include +#include #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -434,8 +451,8 @@ FILE *sqlite3_fopen(const char *zFilename, const char *zMode){ sz1 = (int)strlen(zFilename); sz2 = (int)strlen(zMode); - b1 = sqlite3_malloc( (sz1+1)*sizeof(b1[0]) ); - b2 = sqlite3_malloc( (sz2+1)*sizeof(b1[0]) ); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); if( b1 && b2 ){ sz1 = MultiByteToWideChar(CP_UTF8, 0, zFilename, sz1, b1, sz1); b1[sz1] = 0; @@ -460,8 +477,8 @@ FILE *sqlite3_popen(const char *zCommand, const char *zMode){ sz1 = (int)strlen(zCommand); sz2 = (int)strlen(zMode); - b1 = sqlite3_malloc( (sz1+1)*sizeof(b1[0]) ); - b2 = sqlite3_malloc( (sz2+1)*sizeof(b1[0]) ); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); if( b1 && b2 ){ sz1 = MultiByteToWideChar(CP_UTF8, 0, zCommand, sz1, b1, sz1); b1[sz1] = 0; @@ -484,7 +501,7 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){ ** that into UTF-8. Otherwise, non-ASCII characters all get translated ** into '?'. */ - wchar_t *b1 = sqlite3_malloc( sz*sizeof(wchar_t) ); + wchar_t *b1 = sqlite3_malloc64( sz*sizeof(wchar_t) ); if( b1==0 ) return 0; #ifdef SQLITE_USE_W32_FOR_CONSOLE_IO DWORD nRead = 0; @@ -559,7 +576,7 @@ int sqlite3_fputs(const char *z, FILE *out){ ** to the console on Windows. */ int sz = (int)strlen(z); - wchar_t *b1 = sqlite3_malloc( (sz+1)*sizeof(wchar_t) ); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(wchar_t) ); if( b1==0 ) return 0; sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz); b1[sz] = 0; @@ -667,8 +684,8 @@ void sqlite3_fsetmode(FILE *fp, int mode){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** Header file for the Result-Format or "resfmt" utility library for SQLite. -** See the resfmt.md documentation for additional information. +** Header file for the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. */ #ifndef SQLITE_QRF_H #define SQLITE_QRF_H @@ -700,6 +717,7 @@ struct sqlite3_qrf_spec { short int nScreenWidth; /* Maximum overall table width */ short int nLineLimit; /* Maximum number of lines for any row */ short int nTitleLimit; /* Maximum number of characters in a title */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ int nCharLimit; /* Maximum number of characters in a cell */ int nWidth; /* Number of entries in aWidth[] */ int nAlign; /* Number of entries in aAlignment[] */ @@ -870,8 +888,8 @@ size_t sqlite3_qrf_wcswidth(const char*); ** May you share freely, never taking more than you give. ** ************************************************************************* -** Implementation of the Result-Format or "qrf" utility library for SQLite. -** See the qrf.md documentation for additional information. +** Implementation of the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. */ #ifndef SQLITE_QRF_H #include "qrf.h" @@ -880,7 +898,9 @@ size_t sqlite3_qrf_wcswidth(const char*); #include #include +#ifndef SQLITE_AMALGAMATION /* typedef sqlite3_int64 i64; */ +#endif /* A single line in the EQP output */ typedef struct qrfEQPGraphRow qrfEQPGraphRow; @@ -927,6 +947,7 @@ struct Qrf { int iIndent; /* Current slot */ int *aiIndent; /* Indentation for each opcode */ } sExpln; + unsigned int nIns; /* Bytes used for current INSERT stmt */ } u; sqlite3_int64 nRow; /* Number of rows handled so far */ int *actualWidth; /* Actual width of each column */ @@ -3424,30 +3445,46 @@ static void qrfOneSimpleRow(Qrf *p){ break; } case QRF_STYLE_Insert: { - if( qrf_need_quote(p->spec.zTableName) ){ - sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); - }else{ - sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); - } - if( p->spec.bTitles==QRF_Yes ){ - for(i=0; inCol; i++){ - const char *zCName = sqlite3_column_name(p->pStmt, i); - if( qrf_need_quote(zCName) ){ - sqlite3_str_appendf(p->pOut, "%c\"%w\"", - i==0 ? '(' : ',', zCName); - }else{ - sqlite3_str_appendf(p->pOut, "%c%s", - i==0 ? '(' : ',', zCName); + unsigned int mxIns = p->spec.nMultiInsert; + int szStart = sqlite3_str_length(p->pOut); + if( p->u.nIns==0 || p->u.nIns>=mxIns ){ + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + p->u.nIns = 0; + } + if( qrf_need_quote(p->spec.zTableName) ){ + sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); + }else{ + sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); + } + if( p->spec.bTitles==QRF_Yes ){ + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( qrf_need_quote(zCName) ){ + sqlite3_str_appendf(p->pOut, "%c\"%w\"", + i==0 ? '(' : ',', zCName); + }else{ + sqlite3_str_appendf(p->pOut, "%c%s", + i==0 ? '(' : ',', zCName); + } } + sqlite3_str_append(p->pOut, ")", 1); } - sqlite3_str_append(p->pOut, ")", 1); + sqlite3_str_append(p->pOut," VALUES(", 8); + }else{ + sqlite3_str_append(p->pOut,",\n (", 5); } - sqlite3_str_append(p->pOut," VALUES(", 8); for(i=0; inCol; i++){ if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); qrfRenderValue(p, p->pOut, i); } - sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart; + if( p->u.nIns>=mxIns ){ + sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns = 0; + }else{ + sqlite3_str_append(p->pOut, ")", 1); + } qrfWrite(p); break; } @@ -3491,7 +3528,9 @@ static void qrfOneSimpleRow(Qrf *p){ do{ int nThis, nWide, iNext; qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); - if( cnt ) sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+2,' '); + if( cnt ){ + sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+nSep,' '); + } cnt++; if( cnt>p->mxHeight ){ zVal = "..."; @@ -3554,7 +3593,7 @@ static void qrfInitialize( size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ memset(p, 0, sizeof(*p)); p->pzErr = pzErr; - if( pSpec->iVersion!=1 ){ + if( pSpec->iVersion>1 ){ qrfError(p, SQLITE_ERROR, "unusable sqlite3_qrf_spec.iVersion (%d)", pSpec->iVersion); @@ -3614,6 +3653,7 @@ qrf_reinit: if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ p->spec.zTableName = "tab"; } + p->u.nIns = 0; break; } case QRF_STYLE_Line: { @@ -3712,20 +3752,23 @@ static void qrfFinalize(Qrf *p){ switch( p->spec.eStyle ){ case QRF_STYLE_Count: { sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); - qrfWrite(p); break; } case QRF_STYLE_Json: { if( p->nRow>0 ){ sqlite3_str_append(p->pOut, "}]\n", 3); - qrfWrite(p); } break; } case QRF_STYLE_JObject: { if( p->nRow>0 ){ sqlite3_str_append(p->pOut, "}\n", 2); - qrfWrite(p); + } + break; + } + case QRF_STYLE_Insert: { + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); } break; } @@ -3745,15 +3788,14 @@ static void qrfFinalize(Qrf *p){ SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); #endif qrfEqpRender(p, nCycle); - qrfWrite(p); break; } case QRF_STYLE_Eqp: { qrfEqpRender(p, 0); - qrfWrite(p); break; } } + qrfWrite(p); qrfStrErr(p, p->pOut); if( p->spec.pzOutput ){ if( p->spec.pzOutput[0] ){ @@ -3858,1052 +3900,288 @@ int sqlite3_format_query_result( #endif /* -** Output routines that are able to redirect to memory rather than -** doing actually I/O. -** Works like. -** -------------- -** cli_printf(FILE*, const char*, ...); fprintf() -** cli_puts(const char*, FILE*); fputs() -** cli_vprintf(FILE*, const char*, va_list); vfprintf() -** -** These are just thin wrappers with the following added semantics: -** If the file-scope variable cli_output_capture is not NULL, and -** if the FILE* argument is stdout or stderr, then rather than -** writing to stdout/stdout, append the text to the cli_output_capture -** variable. -** -** The cli_exit(int) routine works like exit() except that it -** first dumps any capture output to stdout. -*/ -static sqlite3_str *cli_output_capture = 0; -static int cli_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - int rc; - va_start(ap,zFormat); - if( cli_output_capture && (out==stdout || out==stderr) ){ - sqlite3_str_vappendf(cli_output_capture, zFormat, ap); - rc = 1; - }else{ - rc = sqlite3_vfprintf(out, zFormat, ap); - } - va_end(ap); - return rc; -} -static int cli_puts(const char *zText, FILE *out){ - if( cli_output_capture && (out==stdout || out==stderr) ){ - sqlite3_str_appendall(cli_output_capture, zText); - return 1; - } - return sqlite3_fputs(zText, out); -} -#if 0 /* Not currently used - available if we need it later */ -static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ - if( cli_output_capture && (out==stdout || out==stderr) ){ - sqlite3_str_vappendf(cli_output_capture, zFormat, ap); - return 1; - }else{ - return sqlite3_vfprintf(out, zFormat, ap); - } -} -#endif -static void cli_exit(int rc){ - if( cli_output_capture ){ - char *z = sqlite3_str_finish(cli_output_capture); - sqlite3_fputs(z, stdout); - fflush(stdout); - } - exit(rc); -} - - -#define eputz(z) cli_puts(z,stderr) -#define sputz(fp,z) cli_puts(z,fp) - -/* A version of strcmp() that works with NULL values */ -static int cli_strcmp(const char *a, const char *b){ - if( a==0 ) a = ""; - if( b==0 ) b = ""; - return strcmp(a,b); -} -static int cli_strncmp(const char *a, const char *b, size_t n){ - if( a==0 ) a = ""; - if( b==0 ) b = ""; - return strncmp(a,b,n); -} - -/* Return the current wall-clock time in microseconds since the -** Unix epoch (1970-01-01T00:00:00Z) +** The source code for several run-time loadable extensions is inserted +** below by the ../tool/mkshellc.tcl script. Before processing that included +** code, we need to override some macros to make the included program code +** work here in the middle of this regular program. */ -static sqlite3_int64 timeOfDay(void){ -#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 - sqlite3_uint64 t; - FILETIME tm; - GetSystemTimePreciseAsFileTime(&tm); - t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; - t += 116444736000000000LL; - t /= 10; - return t; -#elif defined(_WIN32) - static sqlite3_vfs *clockVfs = 0; - sqlite3_int64 t; - if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); - if( clockVfs==0 ) return 0; /* Never actually happens */ - if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ - clockVfs->xCurrentTimeInt64(clockVfs, &t); - }else{ - double r; - clockVfs->xCurrentTime(clockVfs, &r); - t = (sqlite3_int64)(r*86400000.0); - } - return t*1000; -#else - struct timeval sNow; - (void)gettimeofday(&sNow,0); - return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; -#endif -} - +#define SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT2(X) (void)(X) +/************************* Begin ext/misc/windirent.h ******************/ /* -** Used to prevent warnings about unused parameters +** 2025-06-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** An implementation of opendir(), readdir(), and closedir() for Windows, +** based on the FindFirstFile(), FindNextFile(), and FindClose() APIs +** of Win32. +** +** #include this file inside any C-code module that needs to use +** opendir()/readdir()/closedir(). This file is a no-op on non-Windows +** machines. On Windows, static functions are defined that implement +** those standard interfaces. */ -#define UNUSED_PARAMETER(x) (void)(x) +#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) +#define SQLITE_WINDIRENT_H -/* -** Number of elements in an array -*/ -#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef FILENAME_MAX +# define FILENAME_MAX (260) +#endif +#ifndef S_ISREG +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +#endif +#ifndef S_ISDIR +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif +#ifndef S_ISLNK +#define S_ISLNK(m) (0) +#endif +typedef unsigned short mode_t; -/* -** If the following flag is set, then command execution stops -** at an error if we are not interactive. +/* The dirent object for Windows is abbreviated. The only field really +** usable by applications is d_name[]. */ -static int bail_on_error = 0; +struct dirent { + int d_ino; /* Inode number (synthesized) */ + unsigned d_attributes; /* File attributes */ + char d_name[FILENAME_MAX]; /* Null-terminated filename */ +}; -/* -** Treat stdin as an interactive input if the following variable -** is true. Otherwise, assume stdin is connected to a file or pipe. -*/ -static int stdin_is_interactive = 1; +/* The internals of DIR are opaque according to standards. So it +** does not matter what we put here. */ +typedef struct DIR DIR; +struct DIR { + intptr_t d_handle; /* Handle for findfirst()/findnext() */ + struct dirent cur; /* Current entry */ +}; -/* -** Treat stdout like a TTY if true. -*/ -static int stdout_is_console = 1; +/* Ignore hidden and system files */ +#define WindowsFileToIgnore(a) \ + ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) /* -** Use this value as the width of the output device. Or, figure it -** out at runtime if the value is negative. Or use a default width -** if this value is zero. +** Close a previously opened directory */ -static int stdout_tty_width = -1; +static int closedir(DIR *pDir){ + int rc = 0; + if( pDir==0 ){ + return EINVAL; + } + if( pDir->d_handle!=0 && pDir->d_handle!=(-1) ){ + rc = _findclose(pDir->d_handle); + } + sqlite3_free(pDir); + return rc; +} /* -** The following is the open SQLite database. We make a pointer -** to this database a static variable so that it can be accessed -** by the SIGINT handler to interrupt database processing. +** Open a new directory. The directory name should be UTF-8 encoded. +** appropriate translations happen automatically. */ -static sqlite3 *globalDb = 0; +static DIR *opendir(const char *zDirName){ + DIR *pDir; + wchar_t *b1; + sqlite3_int64 sz; + struct _wfinddata_t data; -/* -** True if an interrupt (Control-C) has been received. -*/ -static volatile int seenInterrupt = 0; + pDir = sqlite3_malloc64( sizeof(DIR) ); + if( pDir==0 ) return 0; + memset(pDir, 0, sizeof(DIR)); + memset(&data, 0, sizeof(data)); + sz = strlen(zDirName); + b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); + if( b1==0 ){ + closedir(pDir); + return NULL; + } + sz = MultiByteToWideChar(CP_UTF8, 0, zDirName, sz, b1, sz); + b1[sz++] = '\\'; + b1[sz++] = '*'; + b1[sz] = 0; + if( sz+1>sizeof(data.name)/sizeof(data.name[0]) ){ + closedir(pDir); + sqlite3_free(b1); + return NULL; + } + memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); + sqlite3_free(b1); + pDir->d_handle = _wfindfirst(data.name, &data); + if( pDir->d_handle<0 ){ + closedir(pDir); + return NULL; + } + while( WindowsFileToIgnore(data) ){ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + closedir(pDir); + return NULL; + } + } + pDir->cur.d_ino = 0; + pDir->cur.d_attributes = data.attrib; + WideCharToMultiByte(CP_UTF8, 0, data.name, -1, + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return pDir; +} /* -** This is the name of our program. It is set in main(), used -** in a number of other places, mostly for error messages. +** Read the next entry from a directory. +** +** The returned struct-dirent object is managed by DIR. It is only +** valid until the next readdir() or closedir() call. Only the +** d_name[] field is meaningful. The d_name[] value has been +** translated into UTF8. */ -static char *Argv0; +static struct dirent *readdir(DIR *pDir){ + struct _wfinddata_t data; + if( pDir==0 ) return 0; + if( (pDir->cur.d_ino++)==0 ){ + return &pDir->cur; + } + do{ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + return NULL; + } + }while( WindowsFileToIgnore(data) ); + pDir->cur.d_attributes = data.attrib; + WideCharToMultiByte(CP_UTF8, 0, data.name, -1, + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return &pDir->cur; +} -/* -** Prompt strings. Initialized in main. Settable with -** .prompt main continue -*/ -#define PROMPT_LEN_MAX 128 -/* First line prompt. default: "sqlite> " */ -static char mainPrompt[PROMPT_LEN_MAX]; -/* Continuation prompt. default: " ...> " */ -static char continuePrompt[PROMPT_LEN_MAX]; +#endif /* defined(_WIN32) && defined(_MSC_VER) */ +/************************* End ext/misc/windirent.h ********************/ +/************************* Begin ext/misc/memtrace.c ******************/ /* -** Write I/O traces to the following stream. +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. */ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif +#include +#include +#include +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; -/* This is variant of the standard-library strncpy() routine with the -** one change that the destination string is always zero-terminated, even -** if there is no zero-terminator in the first n-1 characters of the source -** string. -*/ -static char *shell_strncpy(char *dest, const char *src, size_t n){ - size_t i; - for(i=0; i %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); } -/* -** Optionally disable dynamic continuation prompt. -** Unless disabled, the continuation prompt shows open SQL lexemes if any, -** or open parentheses level if non-zero, or continuation prompt as set. -** This facility interacts with the scanner and process_input() where the -** below 5 macros are used. -*/ -#ifdef SQLITE_OMIT_DYNAPROMPT -# define CONTINUATION_PROMPT continuePrompt -# define CONTINUE_PROMPT_RESET -# define CONTINUE_PROMPT_AWAITS(p,s) -# define CONTINUE_PROMPT_AWAITC(p,c) -# define CONTINUE_PAREN_INCR(p,n) -# define CONTINUE_PROMPT_PSTATE 0 -typedef void *t_NoDynaPrompt; -# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt -#else -# define CONTINUATION_PROMPT dynamicContinuePrompt() -# define CONTINUE_PROMPT_RESET \ - do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) -# define CONTINUE_PROMPT_AWAITS(p,s) \ - if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) -# define CONTINUE_PROMPT_AWAITC(p,c) \ - if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) -# define CONTINUE_PAREN_INCR(p,n) \ - if(p && stdin_is_interactive) (trackParenLevel(p,n)) -# define CONTINUE_PROMPT_PSTATE (&dynPrompt) -typedef struct DynaPrompt *t_DynaPromptRef; -# define SCAN_TRACKER_REFTYPE t_DynaPromptRef - -static struct DynaPrompt { - char dynamicPrompt[PROMPT_LEN_MAX]; - char acAwait[2]; - int inParenLevel; - char *zScannerAwaits; -} dynPrompt = { {0}, {0}, 0, 0 }; +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; -/* Record parenthesis nesting level change, or force level to 0. */ -static void trackParenLevel(struct DynaPrompt *p, int ni){ - p->inParenLevel += ni; - if( ni==0 ) p->inParenLevel = 0; - p->zScannerAwaits = 0; +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; } -/* Record that a lexeme is opened, or closed with args==0. */ -static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ - if( s!=0 || c==0 ){ - p->zScannerAwaits = s; - p->acAwait[0] = 0; - }else{ - p->acAwait[0] = c; - p->zScannerAwaits = p->acAwait; +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } } -} - -/* Upon demand, derive the continuation prompt to display. */ -static char *dynamicContinuePrompt(void){ - if( continuePrompt[0]==0 - || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ - return continuePrompt; - }else{ - if( dynPrompt.zScannerAwaits ){ - size_t ncp = strlen(continuePrompt); - size_t ndp = strlen(dynPrompt.zScannerAwaits); - if( ndp > ncp-3 ) return continuePrompt; - shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); - while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; - shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, - PROMPT_LEN_MAX-4); - }else{ - if( dynPrompt.inParenLevel>9 ){ - shell_strncpy(dynPrompt.dynamicPrompt, "(..", 4); - }else if( dynPrompt.inParenLevel<0 ){ - shell_strncpy(dynPrompt.dynamicPrompt, ")x!", 4); - }else{ - shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); - dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); - } - shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, - PROMPT_LEN_MAX-4); - } - } - return dynPrompt.dynamicPrompt; -} -#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ - -/* Indicate out-of-memory and exit. */ -static void shell_out_of_memory(void){ - eputz("Error: out of memory\n"); - cli_exit(1); -} - -/* Check a pointer to see if it is NULL. If it is NULL, exit with an -** out-of-memory error. -*/ -static void shell_check_oom(const void *p){ - if( p==0 ) shell_out_of_memory(); -} - -/* -** This routine works like printf in that its first argument is a -** format string and subsequent arguments are values to be substituted -** in place of % fields. The result of formatting this string -** is written to iotrace. -*/ -#ifdef SQLITE_ENABLE_IOTRACE -static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ - va_list ap; - char *z; - if( iotrace==0 ) return; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - cli_printf(iotrace, "%s", z); - sqlite3_free(z); -} -#endif - -/* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. -*/ -static int strlen30(const char *z){ - size_t n; - if( z==0 ) return 0; - n = strlen(z); - return n>0x3fffffff ? 0x3fffffff : (int)n; -} - -/* -** Return open FILE * if zFile exists, can be opened for read -** and is an ordinary file or a character stream source. -** Otherwise return 0. -*/ -static FILE * openChrSource(const char *zFile){ -#if defined(_WIN32) || defined(WIN32) - struct __stat64 x = {0}; -# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) - /* On Windows, open first, then check the stream nature. This order - ** is necessary because _stat() and sibs, when checking a named pipe, - ** effectively break the pipe as its supplier sees it. */ - FILE *rv = sqlite3_fopen(zFile, "rb"); - if( rv==0 ) return 0; - if( _fstat64(_fileno(rv), &x) != 0 - || !STAT_CHR_SRC(x.st_mode)){ - fclose(rv); - rv = 0; - } - return rv; -#else - struct stat x = {0}; - int rc = stat(zFile, &x); -# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) - if( rc!=0 ) return 0; - if( STAT_CHR_SRC(x.st_mode) ){ - return sqlite3_fopen(zFile, "rb"); - }else{ - return 0; - } -#endif -#undef STAT_CHR_SRC -} - -/* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails, or if the length of the line is longer than about a gigabyte. -** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. -*/ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; - int n = 0; - - while( 1 ){ - if( n+100>nLine ){ - if( nLine>=1073741773 ){ - free(zLine); - return 0; - } - nLine = nLine*2 + 100; - zLine = realloc(zLine, nLine); - shell_check_oom(zLine); - } - if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ - if( n==0 ){ - free(zLine); - return 0; - } - zLine[n] = 0; - break; - } - while( zLine[n] ) n++; - if( n>0 && zLine[n-1]=='\n' ){ - n--; - if( n>0 && zLine[n-1]=='\r' ) n--; - zLine[n] = 0; - break; - } - } - return zLine; -} - -/* -** Retrieve a single line of input text. -** -** If in==0 then read from standard input and prompt before each line. -** If isContinuation is true, then a continuation prompt is appropriate. -** If isContinuation is zero, then the main prompt should be used. -** -** If zPrior is not NULL then it is a buffer from a prior call to this -** routine that can be reused. -** -** The result is stored in space obtained from malloc() and must either -** be freed by the caller or else passed back into this routine via the -** zPrior argument for reuse. -*/ -#ifndef SQLITE_SHELL_FIDDLE -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ - char *zPrompt; - char *zResult; - if( in!=0 ){ - zResult = local_getline(zPrior, in); - }else{ - zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; -#if SHELL_USE_LOCAL_GETLINE - sputz(stdout, zPrompt); - fflush(stdout); - do{ - zResult = local_getline(zPrior, stdin); - zPrior = 0; - /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ - if( zResult==0 ) sqlite3_sleep(50); - }while( zResult==0 && seenInterrupt>0 ); -#else - free(zPrior); - zResult = shell_readline(zPrompt); - while( zResult==0 ){ - /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ - sqlite3_sleep(50); - if( seenInterrupt==0 ) break; - zResult = shell_readline(""); - } - if( zResult && *zResult ) shell_add_history(zResult); -#endif - } - return zResult; -} -#endif /* !SQLITE_SHELL_FIDDLE */ - -/* -** Return the value of a hexadecimal digit. Return -1 if the input -** is not a hex digit. -*/ -static int hexDigitValue(char c){ - if( c>='0' && c<='9' ) return c - '0'; - if( c>='a' && c<='f' ) return c - 'a' + 10; - if( c>='A' && c<='F' ) return c - 'A' + 10; - return -1; -} - -/* -** Interpret zArg as an integer value, possibly with suffixes. -** -** If the value specified by zArg is outside the range of values that -** can be represented using a 64-bit twos-complement integer, then return -** the nearest representable value. -*/ -static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_uint64 v = 0; - static const struct { char *zSuffix; unsigned int iMult; } aMult[] = { - { "KiB", 1024 }, - { "MiB", 1024*1024 }, - { "GiB", 1024*1024*1024 }, - { "KB", 1000 }, - { "MB", 1000000 }, - { "GB", 1000000000 }, - { "K", 1000 }, - { "M", 1000000 }, - { "G", 1000000000 }, - }; - int i; - int isNeg = 0; - if( zArg[0]=='-' ){ - isNeg = 1; - zArg++; - }else if( zArg[0]=='+' ){ - zArg++; - } - if( zArg[0]=='0' && zArg[1]=='x' ){ - int x; - zArg += 2; - while( (x = hexDigitValue(zArg[0]))>=0 ){ - if( v > 0x0fffffffffffffffULL ) goto integer_overflow; - v = (v<<4) + x; - zArg++; - } - }else{ - while( IsDigit(zArg[0]) ){ - if( v>=922337203685477580LL ){ - if( v>922337203685477580LL || zArg[0]>='8' ) goto integer_overflow; - } - v = v*10 + (zArg[0] - '0'); - zArg++; - } - } - for(i=0; i0x7fffffffffffffffULL ) goto integer_overflow; - return isNeg? -(sqlite3_int64)v : (sqlite3_int64)v; -integer_overflow: - return isNeg ? (i64)0x8000000000000000LL : 0x7fffffffffffffffLL; -} - -/* -** A variable length string to which one can append text. -*/ -typedef struct ShellText ShellText; -struct ShellText { - char *zTxt; /* The text */ - i64 n; /* Number of bytes of zTxt[] actually used */ - i64 nAlloc; /* Number of bytes allocated for zTxt[] */ -}; - -/* -** Initialize and destroy a ShellText object -*/ -static void initText(ShellText *p){ - memset(p, 0, sizeof(*p)); -} -static void freeText(ShellText *p){ - sqlite3_free(p->zTxt); - initText(p); -} - -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. -** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. -*/ -static void appendText(ShellText *p, const char *zAppend, char quote){ - i64 len; - i64 i; - i64 nAppend = strlen30(zAppend); - - len = nAppend+p->n+1; - if( quote ){ - len += 2; - for(i=0; izTxt==0 || p->n+len>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + len + 20; - p->zTxt = sqlite3_realloc64(p->zTxt, p->nAlloc); - shell_check_oom(p->zTxt); - } - - if( quote ){ - char *zCsr = p->zTxt+p->n; - *zCsr++ = quote; - for(i=0; in = (i64)(zCsr - p->zTxt); - *zCsr = '\0'; - }else{ - memcpy(p->zTxt+p->n, zAppend, nAppend); - p->n += nAppend; - p->zTxt[p->n] = '\0'; - } -} - -/* -** Attempt to determine if identifier zName needs to be quoted, either -** because it contains non-alphanumeric characters, or because it is an -** SQLite keyword. Be conservative in this estimate: When in doubt assume -** that quoting is required. -** -** Return '"' if quoting is required. Return 0 if no quoting is required. -*/ -static char quoteChar(const char *zName){ - int i; - if( zName==0 ) return '"'; - if( !IsAlpha(zName[0]) && zName[0]!='_' ) return '"'; - for(i=0; zName[i]; i++){ - if( !IsAlnum(zName[i]) && zName[i]!='_' ) return '"'; - } - return sqlite3_keyword_check(zName, i) ? '"' : 0; -} - -/* -** Construct a fake object name and column list to describe the structure -** of the view, virtual table, or table valued function zSchema.zName. -** -** The returned string comes from sqlite3_mprintf() and should be freed -** by the caller using sqlite3_free(). -*/ -static char *shellFakeSchema( - sqlite3 *db, /* The database connection containing the vtab */ - const char *zSchema, /* Schema of the database holding the vtab */ - const char *zName /* The name of the virtual table */ -){ - sqlite3_stmt *pStmt = 0; - char *zSql; - ShellText s; - char cQuote; - char *zDiv = "("; - int nRow = 0; - - zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", - zSchema ? zSchema : "main", zName); - shell_check_oom(zSql); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - initText(&s); - if( zSchema ){ - cQuote = quoteChar(zSchema); - if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; - appendText(&s, zSchema, cQuote); - appendText(&s, ".", 0); - } - cQuote = quoteChar(zName); - appendText(&s, zName, cQuote); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); - nRow++; - appendText(&s, zDiv, 0); - zDiv = ","; - if( zCol==0 ) zCol = ""; - cQuote = quoteChar(zCol); - appendText(&s, zCol, cQuote); - } - appendText(&s, ")", 0); - sqlite3_finalize(pStmt); - if( nRow==0 ){ - freeText(&s); - s.zTxt = 0; - } - return s.zTxt; -} - -/* -** SQL function: strtod(X) -** -** Use the C-library strtod() function to convert string X into a double. -** Used for comparing the accuracy of SQLite's internal text-to-float conversion -** routines against the C-library. -*/ -static void shellStrtod( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - char *z = (char*)sqlite3_value_text(apVal[0]); - UNUSED_PARAMETER(nVal); - if( z==0 ) return; - sqlite3_result_double(pCtx, strtod(z,0)); -} - -/* -** SQL function: dtostr(X) -** -** Use the C-library printf() function to convert real value X into a string. -** Used for comparing the accuracy of SQLite's internal float-to-text conversion -** routines against the C-library. -*/ -static void shellDtostr( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - double r = sqlite3_value_double(apVal[0]); - int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; - char z[400]; - if( n<1 ) n = 1; - if( n>350 ) n = 350; - sprintf(z, "%#+.*e", n, r); - sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); -} - -/* -** SQL function: shell_add_schema(S,X) -** -** Add the schema name X to the CREATE statement in S and return the result. -** Examples: -** -** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); -** -** Also works on -** -** CREATE INDEX -** CREATE UNIQUE INDEX -** CREATE VIEW -** CREATE TRIGGER -** CREATE VIRTUAL TABLE -** -** This UDF is used by the .schema command to insert the schema name of -** attached databases into the middle of the sqlite_schema.sql field. -*/ -static void shellAddSchemaName( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - static const char *aPrefix[] = { - "TABLE", - "INDEX", - "UNIQUE INDEX", - "VIEW", - "TRIGGER", - "VIRTUAL TABLE" - }; - int i = 0; - const char *zIn = (const char*)sqlite3_value_text(apVal[0]); - const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); - const char *zName = (const char*)sqlite3_value_text(apVal[2]); - sqlite3 *db = sqlite3_context_db_handle(pCtx); - UNUSED_PARAMETER(nVal); - if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ - for(i=0; i -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef FILENAME_MAX -# define FILENAME_MAX (260) -#endif -#ifndef S_ISREG -#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) -#endif -#ifndef S_ISDIR -#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#endif -#ifndef S_ISLNK -#define S_ISLNK(m) (0) -#endif -typedef unsigned short mode_t; - -/* The dirent object for Windows is abbreviated. The only field really -** usable by applications is d_name[]. -*/ -struct dirent { - int d_ino; /* Inode number (synthesized) */ - unsigned d_attributes; /* File attributes */ - char d_name[FILENAME_MAX]; /* Null-terminated filename */ -}; - -/* The internals of DIR are opaque according to standards. So it -** does not matter what we put here. */ -typedef struct DIR DIR; -struct DIR { - intptr_t d_handle; /* Handle for findfirst()/findnext() */ - struct dirent cur; /* Current entry */ -}; - -/* Ignore hidden and system files */ -#define WindowsFileToIgnore(a) \ - ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) - -/* -** Close a previously opened directory -*/ -static int closedir(DIR *pDir){ - int rc = 0; - if( pDir==0 ){ - return EINVAL; - } - if( pDir->d_handle!=0 && pDir->d_handle!=(-1) ){ - rc = _findclose(pDir->d_handle); - } - sqlite3_free(pDir); - return rc; -} - -/* -** Open a new directory. The directory name should be UTF-8 encoded. -** appropriate translations happen automatically. -*/ -static DIR *opendir(const char *zDirName){ - DIR *pDir; - wchar_t *b1; - sqlite3_int64 sz; - struct _wfinddata_t data; - - pDir = sqlite3_malloc64( sizeof(DIR) ); - if( pDir==0 ) return 0; - memset(pDir, 0, sizeof(DIR)); - memset(&data, 0, sizeof(data)); - sz = strlen(zDirName); - b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); - if( b1==0 ){ - closedir(pDir); - return NULL; - } - sz = MultiByteToWideChar(CP_UTF8, 0, zDirName, sz, b1, sz); - b1[sz++] = '\\'; - b1[sz++] = '*'; - b1[sz] = 0; - if( sz+1>sizeof(data.name)/sizeof(data.name[0]) ){ - closedir(pDir); - sqlite3_free(b1); - return NULL; - } - memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); - sqlite3_free(b1); - pDir->d_handle = _wfindfirst(data.name, &data); - if( pDir->d_handle<0 ){ - closedir(pDir); - return NULL; - } - while( WindowsFileToIgnore(data) ){ - memset(&data, 0, sizeof(data)); - if( _wfindnext(pDir->d_handle, &data)==-1 ){ - closedir(pDir); - return NULL; - } - } - pDir->cur.d_ino = 0; - pDir->cur.d_attributes = data.attrib; - WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - pDir->cur.d_name, FILENAME_MAX, 0, 0); - return pDir; -} - -/* -** Read the next entry from a directory. -** -** The returned struct-dirent object is managed by DIR. It is only -** valid until the next readdir() or closedir() call. Only the -** d_name[] field is meaningful. The d_name[] value has been -** translated into UTF8. -*/ -static struct dirent *readdir(DIR *pDir){ - struct _wfinddata_t data; - if( pDir==0 ) return 0; - if( (pDir->cur.d_ino++)==0 ){ - return &pDir->cur; - } - do{ - memset(&data, 0, sizeof(data)); - if( _wfindnext(pDir->d_handle, &data)==-1 ){ - return NULL; - } - }while( WindowsFileToIgnore(data) ); - pDir->cur.d_attributes = data.attrib; - WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - pDir->cur.d_name, FILENAME_MAX, 0, 0); - return &pDir->cur; -} - -#endif /* defined(_WIN32) && defined(_MSC_VER) */ - -/************************* End ext/misc/windirent.h ********************/ -/************************* Begin ext/misc/memtrace.c ******************/ -/* -** 2019-01-21 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements an extension that uses the SQLITE_CONFIG_MALLOC -** mechanism to add a tracing layer on top of SQLite. If this extension -** is registered prior to sqlite3_initialize(), it will cause all memory -** allocation activities to be logged on standard output, or to some other -** FILE specified by the initializer. -** -** This file needs to be compiled into the application that uses it. -** -** This extension is used to implement the --memtrace option of the -** command-line shell. -*/ -#include -#include -#include - -/* The original memory allocation routines */ -static sqlite3_mem_methods memtraceBase; -static FILE *memtraceOut; - -/* Methods that trace memory allocations */ -static void *memtraceMalloc(int n){ - if( memtraceOut ){ - fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", - memtraceBase.xRoundup(n)); - } - return memtraceBase.xMalloc(n); -} -static void memtraceFree(void *p){ - if( p==0 ) return; - if( memtraceOut ){ - fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); - } - memtraceBase.xFree(p); -} -static void *memtraceRealloc(void *p, int n){ - if( p==0 ) return memtraceMalloc(n); - if( n==0 ){ - memtraceFree(p); - return 0; - } - if( memtraceOut ){ - fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", - memtraceBase.xSize(p), memtraceBase.xRoundup(n)); - } - return memtraceBase.xRealloc(p, n); -} -static int memtraceSize(void *p){ - return memtraceBase.xSize(p); -} -static int memtraceRoundup(int n){ - return memtraceBase.xRoundup(n); -} -static int memtraceInit(void *p){ - return memtraceBase.xInit(p); -} -static void memtraceShutdown(void *p){ - memtraceBase.xShutdown(p); -} - -/* The substitute memory allocator */ -static sqlite3_mem_methods ersaztMethods = { - memtraceMalloc, - memtraceFree, - memtraceRealloc, - memtraceSize, - memtraceRoundup, - memtraceInit, - memtraceShutdown, - 0 -}; - -/* Begin tracing memory allocations to out. */ -int sqlite3MemTraceActivate(FILE *out){ - int rc = SQLITE_OK; - if( memtraceBase.xMalloc==0 ){ - rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); - if( rc==SQLITE_OK ){ - rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); - } - } - memtraceOut = out; - return rc; -} - -/* Deactivate memory tracing */ -int sqlite3MemTraceDeactivate(void){ - int rc = SQLITE_OK; - if( memtraceBase.xMalloc!=0 ){ - rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); - if( rc==SQLITE_OK ){ - memset(&memtraceBase, 0, sizeof(memtraceBase)); - } - } - memtraceOut = 0; - return rc; + memtraceOut = 0; + return rc; } /************************* End ext/misc/memtrace.c ********************/ @@ -6198,16 +5476,19 @@ static void sha1Func( SHA1Context cx; int eType = sqlite3_value_type(argv[0]); int nByte = sqlite3_value_bytes(argv[0]); + const unsigned char *pData; char zOut[44]; assert( argc==1 ); if( eType==SQLITE_NULL ) return; hash_init(&cx); if( eType==SQLITE_BLOB ){ - hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_blob(argv[0]); }else{ - hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_text(argv[0]); } + if( pData==0 ) return; + hash_step(&cx, pData, nByte); if( sqlite3_user_data(context)!=0 ){ /* sha1b() - binary result */ hash_finish(&cx, zOut, 1); @@ -6269,6 +5550,7 @@ static void sha1QueryFunc( } nCol = sqlite3_column_count(pStmt); z = sqlite3_sql(pStmt); + if( z==0 ) z = ""; n = (int)strlen(z); hash_step_vformat(&cx,"S%d:",n); hash_step(&cx,(unsigned char*)z,n); @@ -6492,6 +5774,10 @@ SQLITE_EXTENSION_INIT1 #define IsSpace(X) isspace((unsigned char)X) #endif +#ifndef SQLITE_DECIMAL_MAX_DIGIT +# define SQLITE_DECIMAL_MAX_DIGIT 10000000 +#endif + /* A decimal object */ typedef struct Decimal Decimal; struct Decimal { @@ -6530,7 +5816,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ int i; int iExp = 0; - p = sqlite3_malloc( sizeof(*p) ); + if( zIn==0 ) goto new_from_text_failed; + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) goto new_from_text_failed; p->sign = 0; p->oom = 0; @@ -6589,9 +5876,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + (sqlite3_int64)iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + if( a==0 ) goto new_from_text_failed; + p->a = a; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; } @@ -6609,9 +5897,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + (sqlite3_int64)iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + if( a==0 ) goto new_from_text_failed; + p->a = a; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); p->nDigit += iExp; @@ -6622,6 +5911,7 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ for(i=0; inDigit && p->a[i]==0; i++){} if( i>=p->nDigit ) p->sign = 0; } + if( p->nDigit>SQLITE_DECIMAL_MAX_DIGIT ) goto new_from_text_failed; return p; new_from_text_failed: @@ -6759,6 +6049,8 @@ static void decimal_round(Decimal *p, int N){ int i; int nZero; if( N<1 ) return; + if( p==0 ) return; + if( p->nDigit<=N ) return; for(nZero=0; nZeronDigit && p->a[nZero]==0; nZero++){} N += nZero; if( p->nDigit<=N ) return; @@ -6916,15 +6208,18 @@ cmp_done: static void decimal_expand(Decimal *p, int nDigit, int nFrac){ int nAddSig; int nAddFrac; + signed char *a; if( p==0 ) return; nAddFrac = nFrac - p->nFrac; nAddSig = (nDigit - p->nDigit) - nAddFrac; if( nAddFrac==0 && nAddSig==0 ) return; - p->a = sqlite3_realloc64(p->a, nDigit+1); - if( p->a==0 ){ + if( nDigit+1>SQLITE_DECIMAL_MAX_DIGIT ){ p->oom = 1; return; } + a = sqlite3_realloc64(p->a, nDigit+1); + if( a==0 ){ p->oom = 1; return; } + p->a = a; if( nAddSig ){ memmove(p->a+nAddSig, p->a, p->nDigit); memset(p->a, 0, nAddSig); @@ -7019,14 +6314,18 @@ static void decimalMul(Decimal *pA, Decimal *pB){ signed char *acc = 0; int i, j, k; int minFrac; + sqlite3_int64 sumDigit; if( pA==0 || pA->oom || pA->isNull || pB==0 || pB->oom || pB->isNull ){ goto mul_end; } - acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + - (sqlite3_int64)pB->nDigit + 2 ); + sumDigit = pA->nDigit; + sumDigit += pB->nDigit; + sumDigit += 2; + if( sumDigit>SQLITE_DECIMAL_MAX_DIGIT ){ pA->oom = 1; return; } + acc = sqlite3_malloc64( sumDigit ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -7258,7 +6557,7 @@ static void decimalSumStep( if( p==0 ) return; if( !p->isInit ){ p->isInit = 1; - p->a = sqlite3_malloc(2); + p->a = sqlite3_malloc64(2); if( p->a==0 ){ p->oom = 1; }else{ @@ -7396,8 +6695,6 @@ int sqlite3_decimal_init( } /************************* End ext/misc/decimal.c ********************/ -#undef sqlite3_base_init -#define sqlite3_base_init sqlite3_base64_init /************************* Begin ext/misc/base64.c ******************/ /* ** 2022-11-18 @@ -7633,7 +6930,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_text(context,"",-1,SQLITE_STATIC); break; } - cBuf = sqlite3_malloc(nc); + cBuf = sqlite3_malloc64(nc); if( !cBuf ) goto memFail; nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); @@ -7655,7 +6952,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_zeroblob(context, 0); break; } - bBuf = sqlite3_malloc(nb); + bBuf = sqlite3_malloc64(nb); if( !bBuf ) goto memFail; nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); @@ -7676,7 +6973,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 #endif -int sqlite3_base_init +int sqlite3_base64_init #else static int sqlite3_base64_init #endif @@ -7698,9 +6995,6 @@ static int sqlite3_base64_init #define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ /************************* End ext/misc/base64.c ********************/ -#undef sqlite3_base_init -#define sqlite3_base_init sqlite3_base85_init -#define OMIT_BASE85_CHECKER /************************* Begin ext/misc/base85.c ******************/ /* ** 2022-11-16 @@ -7966,7 +7260,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER /* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); @@ -7986,7 +7280,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ return; } } -# endif +#endif /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ @@ -8013,7 +7307,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_text(context,"",-1,SQLITE_STATIC); break; } - cBuf = sqlite3_malloc(nc); + cBuf = sqlite3_malloc64(nc); if( !cBuf ) goto memFail; nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); @@ -8035,7 +7329,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_zeroblob(context, 0); break; } - bBuf = sqlite3_malloc(nb); + bBuf = sqlite3_malloc64(nb); if( !bBuf ) goto memFail; nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); @@ -8056,14 +7350,14 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 #endif -int sqlite3_base_init +int sqlite3_base85_init #else static int sqlite3_base85_init #endif (sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER { int rc = sqlite3_create_function (db, "is_base85", 1, @@ -8071,7 +7365,7 @@ static int sqlite3_base85_init 0, is_base85, 0, 0); if( rc!=SQLITE_OK ) return rc; } -# endif +#endif return sqlite3_create_function (db, "base85", 1, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, @@ -8136,9 +7430,9 @@ int main(int na, char *av[]){ int nc = strlen(cBuf); size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER b85Clean &= allBase85( cBuf, nc ); -# endif +#endif } break; default: @@ -8764,7 +8058,7 @@ static int seriesConnect( rc = sqlite3_declare_vtab(db, "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = *ppVtab = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); @@ -8786,7 +8080,7 @@ static int seriesDisconnect(sqlite3_vtab *pVtab){ static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ series_cursor *pCur; (void)pUnused; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; @@ -10141,7 +9435,7 @@ static const char *re_compile( int i, j; *ppRe = 0; - pRe = sqlite3_malloc( sizeof(*pRe) ); + pRe = sqlite3_malloc64( sizeof(*pRe) ); if( pRe==0 ){ return "out of memory"; } @@ -10293,7 +9587,6 @@ static void re_bytecode_func( int i; int n; char *z; - (void)argc; static const char *ReOpName[] = { "EOF", "MATCH", @@ -10316,6 +9609,7 @@ static void re_bytecode_func( "ATSTART", }; + (void)argc; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; zErr = re_compile(&pRe, zPattern, re_maxnfa(re_maxlen(context)), @@ -10470,11 +9764,6 @@ int sqlite3_regexp_init( ** $path is a relative path, then $path is interpreted relative to $dir. ** And the paths returned in the "name" column of the table are also ** relative to directory $dir. -** -** Notes on building this extension for Windows: -** Unless linked statically with the SQLite library, a preprocessor -** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone -** DLL form of this extension for WIN32. See its use below for details. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 @@ -11015,7 +10304,7 @@ static int fsdirConnect( (void)pzErr; rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + pNew = (fsdir_tab*)sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); @@ -11038,7 +10327,7 @@ static int fsdirDisconnect(sqlite3_vtab *pVtab){ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ fsdir_cursor *pCur; (void)p; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->iLvl = -1; @@ -11771,7 +11060,7 @@ static int completionConnect( " phase INT HIDDEN" /* Used for debugging only */ ")"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -11793,7 +11082,7 @@ static int completionDisconnect(sqlite3_vtab *pVtab){ */ static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ completion_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((completion_vtab*)p)->db; @@ -11838,6 +11127,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ completion_cursor *pCur = (completion_cursor*)cur; int eNextPhase = 0; /* Next phase to try if current phase reaches end */ int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + int rc; pCur->iRowid++; while( pCur->ePhase!=COMPLETION_EOF ){ switch( pCur->ePhase ){ @@ -11863,22 +11153,27 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_TABLES: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str* pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT name FROM \"%w\".sqlite_schema", - zSql, zSep, zDb + zSep, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_COLUMNS; @@ -11887,24 +11182,29 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_COLUMNS: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str *pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" " WHERE sm.type='table'", - zSql, zSep, zDb, zDb + zSep, zDb, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_EOF; @@ -11921,9 +11221,10 @@ static int completionNext(sqlite3_vtab_cursor *cur){ pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); }else{ /* When all rows are finished, advance to the next phase */ - sqlite3_finalize(pCur->pStmt); + rc = sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; pCur->ePhase = eNextPhase; + if( rc ) return rc; continue; } } @@ -13285,7 +12586,7 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ZipfileTab *pTab = (ZipfileTab*)p; ZipfileCsr *pCsr; - pCsr = sqlite3_malloc(sizeof(*pCsr)); + pCsr = sqlite3_malloc64(sizeof(*pCsr)); *ppCsr = (sqlite3_vtab_cursor*)pCsr; if( pCsr==0 ){ return SQLITE_NOMEM; @@ -13824,7 +13125,7 @@ static void zipfileInflate( int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ ){ - u8 *aRes = sqlite3_malloc(nOut); + u8 *aRes = sqlite3_malloc64(nOut); if( aRes==0 ){ sqlite3_result_error_nomem(pCtx); }else{ @@ -14221,7 +13522,7 @@ static int zipfileBestIndex( static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + pNew = sqlite3_malloc64(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); pNew->cds.zFile = sqlite3_mprintf("%s", zPath); @@ -15171,7 +14472,7 @@ static void sqlarCompressFunc( uLongf nOut = compressBound(nData); Bytef *pOut; - pOut = (Bytef*)sqlite3_malloc(nOut); + pOut = (Bytef*)sqlite3_malloc64(nOut); if( pOut==0 ){ sqlite3_result_error_nomem(context); return; @@ -15209,14 +14510,14 @@ static void sqlarUncompressFunc( sqlite3_int64 sz; assert( argc==2 ); - sz = sqlite3_value_int(argv[1]); + sz = sqlite3_value_int64(argv[1]); if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); - Bytef *pOut = sqlite3_malloc(sz); + Bytef *pOut = sqlite3_malloc64(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ @@ -18837,7 +18138,7 @@ static void stmtrandFunc( p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); if( p==0 ){ unsigned int seed; - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ sqlite3_result_error_nomem(context); return; @@ -19777,7 +19078,7 @@ static int vfstraceOpen( vfstrace_printf(pInfo, "%s.xOpen(%s,flags=0x%x)", pInfo->zVfsName, p->zFName, flags); if( p->pReal->pMethods ){ - sqlite3_io_methods *pNew = sqlite3_malloc( sizeof(*pNew) ); + sqlite3_io_methods *pNew = sqlite3_malloc64( sizeof(*pNew) ); const sqlite3_io_methods *pSub = p->pReal->pMethods; memset(pNew, 0, sizeof(*pNew)); pNew->iVersion = pSub->iVersion; @@ -21796,2820 +21097,3597 @@ static void recoverBitmapFree(RecoverBitmap *pMap){ sqlite3_free(pMap); } -/* -** Set the bit associated with page iPg in bitvec pMap. -*/ -static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ - if( iPg<=pMap->nPg ){ - int iElem = (iPg / 32); - int iBit = (iPg % 32); - pMap->aElem[iElem] |= (((u32)1) << iBit); +/* +** Set the bit associated with page iPg in bitvec pMap. +*/ +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} + +/* +** Query bitmap object pMap for the state of the bit associated with page +** iPg. Return 1 if it is set, or 0 otherwise. +*/ +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg && iPg>0 ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} + +/* +** Set the recover handle error to the error code and message returned by +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database +** handle db. +*/ +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, it attempts to prepare the SQL statement in zSql against +** database handle db. If successful, the statement handle is returned. +** Or, if an error occurs, NULL is returned and an error left in the +** recover handle. +*/ +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zFmt is used as a printf() style format string, +** along with any trailing arguments, to create an SQL statement. This +** SQL statement is prepared against database handle db and, if successful, +** the statment handle returned. Or, if an error occurs - either during +** the printf() formatting or when preparing the resulting SQL - an +** error code and message are left in the recover handle. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } + } + return pStmt; +} + +/* +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +** +** This function returns a copy of the statement handle pointer passed +** as the second argument. +*/ +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); + } + return pStmt; +} + +/* +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +*/ +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); + } +} + +/* +** Run a single SQL statement in zSql. If zSql contains two or more +** SQL statements separated by ';', only the first is run. +** +** Return the sqlite3_finalizer() or sqlite3_prepare() result code +** from running the zSql statement. +*/ +static int recoverOneStmt(sqlite3 *db, const char *zSql){ + sqlite3_stmt *pStmt = 0; + int rc; + if( zSql==0 ) return SQLITE_OK; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ){ + sqlite3_finalize(pStmt); + return rc; + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){} + return sqlite3_finalize(pStmt); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this +** case. +** +** Otherwise, execute a single SQL statment in zSql. Even if zSql contains +** two or more SQL statements separated by ';', only execute the first one. +** If successful, return SQLITE_OK. Or, if an error occurs, leave an error +** code and message in the recover handle and return a copy of the error code. +*/ +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = recoverOneStmt(db, zSql); + if( rc ){ + recoverDbError(p, db); + } + } + return p->errCode; +} + +/* +** Bind the value pVal to parameter iBind of statement pStmt. Leave an +** error in the recover handle passed as the first argument if an error +** (e.g. an OOM) occurs. +*/ +static void recoverBindValue( + sqlite3_recover *p, + sqlite3_stmt *pStmt, + int iBind, + sqlite3_value *pVal +){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_bind_value(pStmt, iBind, pVal); + if( rc ) recoverError(p, rc, 0); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** +** Otherwise, an attempt is made to interpret zFmt as a printf() style +** formatting string and the result of using the trailing arguments for +** parameter substitution with it written into a buffer obtained from +** sqlite3_malloc(). If successful, a pointer to the buffer is returned. +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** Or, if an error occurs, an error code and message is left in the recover +** handle and NULL returned. +*/ +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( p->errCode==SQLITE_OK ){ + if( z==0 ) p->errCode = SQLITE_NOMEM; + }else{ + sqlite3_free(z); + z = 0; + } + return z; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** +** Otherwise, execute "PRAGMA page_count" against the input database. If +** successful, return the integer result. Or, if an error occurs, leave an +** error code and error message in the sqlite3_recover handle and return +** zero. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt ){ + sqlite3_step(pStmt); + nPg = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(p, pStmt); + } + return nPg; +} + +/* +** Implementation of SQL scalar function "read_i32". The first argument to +** this function must be a blob. The second a non-negative integer. This +** function reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +** +** SELECT read_i32(, ) +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]) & 0xFFFF; + + if( (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Implementation of SQL scalar function "page_is_used". This function +** is used as part of the procedure for locating orphan rows for the +** lost-and-found table, and it depends on those routines having populated +** the sqlite3_recover.laf.pUsed variable. +** +** The only argument to this function is a page-number. It returns true +** if the page has already been used somehow during data recovery, or false +** otherwise. +** +** SELECT page_is_used(); +*/ +static void recoverPageIsUsed( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + assert( nArg==1 ); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); +} + +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the argument +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + i64 nPg = recoverPageCount(p); + sqlite3_result_int64(pCtx, nPg); + return; + }else{ + if( p->pGetPage==0 ){ + pStmt = p->pGetPage = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else if( p->errCode==SQLITE_OK ){ + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const u8 *aPg; + int nPg; + assert( p->errCode==SQLITE_OK ); + aPg = sqlite3_column_blob(pStmt, 0); + nPg = sqlite3_column_bytes(pStmt, 0); + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + aPg = p->pPage1Disk; + } + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); + } + recoverReset(p, pStmt); + } + } + + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); + } +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *recoverUnusedString( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Implementation of scalar SQL function "escape_crlf". The argument passed to +** this function is the output of built-in function quote(). If the first +** character of the input is "'", indicating that the value passed to quote() +** was a text value, then this function searches the input for "\n" and "\r" +** characters and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy of the input +** is returned. +*/ +static void recoverEscapeCrlf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + (void)argc; + if( zText && zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } } -} -/* -** Query bitmap object pMap for the state of the bit associated with page -** iPg. Return 1 if it is set, or 0 otherwise. -*/ -static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ - int ret = 1; - if( iPg<=pMap->nPg && iPg>0 ){ - int iElem = (iPg / 32); - int iBit = (iPg % 32); - ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; - } - return ret; + sqlite3_result_value(context, argv[0]); } /* -** Set the recover handle error to the error code and message returned by -** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database -** handle db. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. */ -static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ - return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); } /* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. ** -** Otherwise, it attempts to prepare the SQL statement in zSql against -** database handle db. If successful, the statement handle is returned. -** Or, if an error occurs, NULL is returned and an error left in the -** recover handle. +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). */ -static sqlite3_stmt *recoverPrepare( - sqlite3_recover *p, - sqlite3 *db, - const char *zSql -){ - sqlite3_stmt *pStmt = 0; - if( p->errCode==SQLITE_OK ){ - if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ - recoverDbError(p, db); +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); } } - return pStmt; } /* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). +** Transfer the following settings from the input database to the output +** database: ** -** Otherwise, argument zFmt is used as a printf() style format string, -** along with any trailing arguments, to create an SQL statement. This -** SQL statement is prepared against database handle db and, if successful, -** the statment handle returned. Or, if an error occurs - either during -** the printf() formatting or when preparing the resulting SQL - an -** error code and message are left in the recover handle. +** + page-size, +** + auto-vacuum settings, +** + database encoding, +** + user-version (PRAGMA user_version), and +** + application-id (PRAGMA application_id), and */ -static sqlite3_stmt *recoverPreparePrintf( - sqlite3_recover *p, - sqlite3 *db, - const char *zFmt, ... -){ - sqlite3_stmt *pStmt = 0; +static void recoverTransferSettings(sqlite3_recover *p){ + const char *aPragma[] = { + "encoding", + "page_size", + "auto_vacuum", + "user_version", + "application_id" + }; + int ii; + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ if( p->errCode==SQLITE_OK ){ - va_list ap; - char *z; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - p->errCode = SQLITE_NOMEM; - }else{ - pStmt = recoverPrepare(p, db, z); - sqlite3_free(z); + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc!=SQLITE_OK ){ + recoverDbError(p, db2); + return; + } + + for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){ + const char *zPrag = aPragma[ii]; + sqlite3_stmt *p1 = 0; + p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag); + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ + const char *zArg = (const char*)sqlite3_column_text(p1, 0); + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); + recoverSqlCallback(p, z2); + recoverExec(p, db2, z2); + sqlite3_free(z2); + if( zArg==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + recoverFinalize(p, p1); + } + recoverExec(p, db2, "CREATE TABLE t1(a)"); + recoverExec(p, db2, "DROP TABLE t1"); + + if( p->errCode==SQLITE_OK ){ + sqlite3 *db = p->dbOut; + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + sqlite3_backup_step(pBackup, -1); + p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); + } } + + sqlite3_close(db2); } - return pStmt; } /* -** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() -** indicates that an error occurred, and there is not already an error -** in the recover handle passed as the first argument, set the error -** code and error message appropriately. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. ** -** This function returns a copy of the statement handle pointer passed -** as the second argument. +** Otherwise, an attempt is made to open the output database, attach +** and create the schema of the temporary database used to store +** intermediate data, and to register all required user functions and +** virtual table modules with the output handle. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. */ -static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ - int rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ - recoverDbError(p, sqlite3_db_handle(pStmt)); - } - return pStmt; -} +static int recoverOpenOutput(sqlite3_recover *p){ + struct Func { + const char *zName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFunc[] = { + { "getpage", 1, recoverGetPage }, + { "page_is_used", 1, recoverPageIsUsed }, + { "read_i32", 2, recoverReadI32 }, + { "escape_crlf", 1, recoverEscapeCrlf }, + }; -/* -** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() -** indicates that an error occurred, and there is not already an error -** in the recover handle passed as the first argument, set the error -** code and error message appropriately. -*/ -static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ - sqlite3 *db = sqlite3_db_handle(pStmt); - int rc = sqlite3_finalize(pStmt); - if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; /* New database handle */ + int ii; /* For iterating through aFunc[] */ + + assert( p->dbOut==0 ); + + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ recoverDbError(p, db); } -} -/* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this -** case. -** -** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. -** Or, if an error occurs, leave an error code and message in the recover -** handle and return a copy of the error code. -*/ -static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. + ** These two are registered with the output database handle - this + ** module depends on the input handle supporting the sqlite_dbpage + ** virtual table only. */ if( p->errCode==SQLITE_OK ){ - int rc = sqlite3_exec(db, zSql, 0, 0, 0); - if( rc ){ - recoverDbError(p, db); - } + p->errCode = sqlite3_dbdata_init(db, 0, 0); + } + + /* Register the custom user-functions with the output handle. */ + for(ii=0; + p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0])); + ii++){ + p->errCode = sqlite3_create_function(db, aFunc[ii].zName, + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + ); } + + p->dbOut = db; return p->errCode; } /* -** Bind the value pVal to parameter iBind of statement pStmt. Leave an -** error in the recover handle passed as the first argument if an error -** (e.g. an OOM) occurs. +** Attach the auxiliary database 'recovery' to the output database handle. +** This temporary database is used during the recovery process and then +** discarded. */ -static void recoverBindValue( - sqlite3_recover *p, - sqlite3_stmt *pStmt, - int iBind, - sqlite3_value *pVal -){ - if( p->errCode==SQLITE_OK ){ - int rc = sqlite3_bind_value(pStmt, iBind, pVal); - if( rc ) recoverError(p, rc, 0); - } +static void recoverOpenRecovery(sqlite3_recover *p){ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, p->dbOut, zSql); + sqlite3_free(zSql); + recoverExec(p, p->dbOut, "PRAGMA writable_schema = 1"); + recoverExec(p, p->dbOut, + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT)"); + recoverExec(p, p->dbOut, + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql)"); } + /* ** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** (if p->errCode!=SQLITE_OK). ** -** Otherwise, an attempt is made to interpret zFmt as a printf() style -** formatting string and the result of using the trailing arguments for -** parameter substitution with it written into a buffer obtained from -** sqlite3_malloc(). If successful, a pointer to the buffer is returned. -** It is the responsibility of the caller to eventually free the buffer -** using sqlite3_free(). +** Otherwise, argument zName must be the name of a table that has just been +** created in the output database. This function queries the output db +** for the schema of said table, and creates a RecoverTable object to +** store the schema in memory. The new RecoverTable object is linked into +** the list at sqlite3_recover.pTblList. ** -** Or, if an error occurs, an error code and message is left in the recover -** handle and NULL returned. +** Parameter iRoot must be the root page of table zName in the INPUT +** database. */ -static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ - va_list ap; - char *z; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( p->errCode==SQLITE_OK ){ - if( z==0 ) p->errCode = SQLITE_NOMEM; - }else{ - sqlite3_free(z); - z = 0; +static void recoverAddTable( + sqlite3_recover *p, + const char *zName, /* Name of table created in output db */ + i64 iRoot /* Root page of same table in INPUT db */ +){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); + + if( pStmt ){ + int iPk = -1; + int iBind = 1; + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + int iField = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int iPKF = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; + if( iPKF>1 ) iPk = -2; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ + pNew->aCol[i].iField = -1; + }else{ + pNew->aCol[i].iField = iField++; + } + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + pNew->aCol[i].iBind = iBind++; + } + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + pNew->bIntkey = 1; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + int iField = sqlite3_column_int(pStmt, 0); + int iCol = sqlite3_column_int(pStmt, 1); + + assert( iColnCol ); + pNew->aCol[iCol].iField = iField; + + pNew->bIntkey = 0; + iPk = -2; + } + recoverFinalize(p, pStmt); + + if( p->errCode==SQLITE_OK ){ + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } + } } - return z; } /* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** This function is called after recoverCacheSchema() has cached those parts +** of the input database schema that could be recovered in temporary table +** "recovery.schema". This function creates in the output database copies +** of all parts of that schema that must be created before the tables can +** be populated. Specifically, this means: ** -** Otherwise, execute "PRAGMA page_count" against the input database. If -** successful, return the integer result. Or, if an error occurs, leave an -** error code and error message in the sqlite3_recover handle and return -** zero. +** * all tables that are not VIRTUAL, and +** * UNIQUE indexes. +** +** If the recovery handle uses SQL callbacks, then callbacks containing +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. +** +** Additionally, records are added to the sqlite_schema table of the +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE +** records are written directly to sqlite_schema, not actually executed. +** If the handle is in SQL callback mode, then callbacks are invoked +** with equivalent SQL statements. */ -static i64 recoverPageCount(sqlite3_recover *p){ - i64 nPg = 0; - if( p->errCode==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); - if( pStmt ){ - sqlite3_step(pStmt); - nPg = sqlite3_column_int64(pStmt, 0); +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" + " SELECT rootpage, name, sql, " + " type='table', " + " sql LIKE 'create virtual%'," + " (type='index' AND (sql LIKE '%unique%' OR ?1))" + " FROM recovery.schema" + ")" + "SELECT rootpage, tbl, isVirtual, name, sql" + " FROM dbschema " + " WHERE (tbl OR isIndex) AND sql GLOB 'CREATE *'" + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" + ); + + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); + + if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + int bTable = sqlite3_column_int(pSelect, 1); + int bVirtual = sqlite3_column_int(pSelect, 2); + const char *zName = (const char*)sqlite3_column_text(pSelect, 3); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); + char *zFree = 0; + int rc = SQLITE_OK; + + if( bVirtual ){ + zSql = (const char*)(zFree = recoverMPrintf(p, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + )); + } + rc = recoverOneStmt(p->dbOut, zSql); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + if( bTable && !bVirtual ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + sqlite3_free(zFree); } - recoverFinalize(p, pStmt); } - return nPg; + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; } /* -** Implementation of SQL scalar function "read_i32". The first argument to -** this function must be a blob. The second a non-negative integer. This -** function reads and returns a 32-bit big-endian integer from byte -** offset (4*) of the blob. +** This function is called after the output database has been populated. It +** adds all recovered schema elements that were not created in the output +** database by recoverWriteSchema1() - everything except for tables and +** UNIQUE indexes. Specifically: ** -** SELECT read_i32(, ) +** * views, +** * triggers, +** * non-UNIQUE indexes. +** +** If the recover handle is in SQL callback mode, then equivalent callbacks +** are issued to create the schema elements. */ -static void recoverReadI32( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *pBlob; - int nBlob; - int iInt; +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; - assert( argc==2 ); - nBlob = sqlite3_value_bytes(argv[0]); - pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); - iInt = sqlite3_value_int(argv[1]) & 0xFFFF; + pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + " AND sql GLOB 'CREATE *'" + : + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + " AND sql GLOB 'CREATE *'" + ); - if( (iInt+1)*4<=nBlob ){ - const unsigned char *a = &pBlob[iInt*4]; - i64 iVal = ((i64)a[0]<<24) - + ((i64)a[1]<<16) - + ((i64)a[2]<< 8) - + ((i64)a[3]<< 0); - sqlite3_result_int64(context, iVal); + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = recoverOneStmt(p->dbOut, zSql); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } } + recoverFinalize(p, pSelect); + + return p->errCode; } /* -** Implementation of SQL scalar function "page_is_used". This function -** is used as part of the procedure for locating orphan rows for the -** lost-and-found table, and it depends on those routines having populated -** the sqlite3_recover.laf.pUsed variable. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). In this case it returns NULL. ** -** The only argument to this function is a page-number. It returns true -** if the page has already been used somehow during data recovery, or false -** otherwise. +** Otherwise, if the recover handle is configured to create an output +** database (was created by sqlite3_recover_init()), then this function +** prepares and returns an SQL statement to INSERT a new record into table +** pTab, assuming the first nField fields of a record extracted from disk +** are valid. ** -** SELECT page_is_used(); -*/ -static void recoverPageIsUsed( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); - i64 pgno = sqlite3_value_int64(apArg[0]); - assert( nArg==1 ); - sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); -} - -/* -** The implementation of a user-defined SQL function invoked by the -** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages -** of the database being recovered. +** For example, if table pTab is: ** -** This function always takes a single integer argument. If the argument -** is zero, then the value returned is the number of pages in the db being -** recovered. If the argument is greater than zero, it is a page number. -** The value returned in this case is an SQL blob containing the data for -** the identified page of the db being recovered. e.g. +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); ** -** SELECT getpage(0); -- return number of pages in db -** SELECT getpage(4); -- return page 4 of db as a blob of data +** And nField is 4, then the SQL statement prepared and returned is: +** +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); +** +** In this case even though 4 values were extracted from the input db, +** only 3 are written to the output, as the generated STORED column +** cannot be written. +** +** If the recover handle is in SQL callback mode, then the SQL statement +** prepared is such that evaluating it returns a single row containing +** a single text value - itself an SQL statement similar to the above, +** except with SQL literals in place of the variables. For example: +** +** SELECT 'INSERT INTO (a, c, d) VALUES (' +** || quote(?1) || ', ' +** || quote(?2) || ', ' +** || quote(?3) || ')'; +** +** In either case, it is the responsibility of the caller to eventually +** free the statement handle using sqlite3_finalize(). */ -static void recoverGetPage( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField ){ - sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); - i64 pgno = sqlite3_value_int64(apArg[0]); - sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pRet = 0; + const char *zSep = ""; + const char *zSqlSep = ""; + char *zSql = 0; + char *zFinal = 0; + char *zBind = 0; + int ii; + int bSql = p->xSql ? 1 : 0; - assert( nArg==1 ); - if( pgno==0 ){ - i64 nPg = recoverPageCount(p); - sqlite3_result_int64(pCtx, nPg); - return; - }else{ - if( p->pGetPage==0 ){ - pStmt = p->pGetPage = recoverPreparePrintf( - p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb - ); - }else if( p->errCode==SQLITE_OK ){ - pStmt = p->pGetPage; + if( nField<=0 ) return 0; + + assert( nField<=pTab->nCol ); + + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + + if( pTab->iRowidBind ){ + assert( pTab->bIntkey ); + zSql = recoverMPrintf(p, "%z_rowid_", zSql); + if( bSql ){ + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); + }else{ + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); } + zSqlSep = "||', '||"; + zSep = ", "; + } - if( pStmt ){ - sqlite3_bind_int64(pStmt, 1, pgno); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - const u8 *aPg; - int nPg; - assert( p->errCode==SQLITE_OK ); - aPg = sqlite3_column_blob(pStmt, 0); - nPg = sqlite3_column_bytes(pStmt, 0); - if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ - aPg = p->pPage1Disk; - } - sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + + if( bSql ){ + zBind = recoverMPrintf(p, + "%z%sescape_crlf(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind + ); + zSqlSep = "||', '||"; + }else{ + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); } - recoverReset(p, pStmt); + zSep = ", "; } } - if( p->errCode ){ - if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); - sqlite3_result_error_code(pCtx, p->errCode); + if( bSql ){ + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", + zSql, zBind + ); + }else{ + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); } + + pRet = recoverPrepare(p, p->dbOut, zFinal); + sqlite3_free(zSql); + sqlite3_free(zBind); + sqlite3_free(zFinal); + + return pRet; } + /* -** Find a string that is not found anywhere in z[]. Return a pointer -** to that string. -** -** Try to use zA and zB first. If both of those are already found in z[] -** then make up some string and store it in the buffer zBuf. +** Search the list of RecoverTable objects at p->pTblList for one that +** has root page iRoot in the input database. If such an object is found, +** return a pointer to it. Otherwise, return NULL. */ -static const char *recoverUnusedString( - const char *z, /* Result must not appear anywhere in z */ - const char *zA, const char *zB, /* Try these first */ - char *zBuf /* Space to store a generated string */ -){ - unsigned i = 0; - if( strstr(z, zA)==0 ) return zA; - if( strstr(z, zB)==0 ) return zB; - do{ - sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); - }while( strstr(z,zBuf)!=0 ); - return zBuf; +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; } /* -** Implementation of scalar SQL function "escape_crlf". The argument passed to -** this function is the output of built-in function quote(). If the first -** character of the input is "'", indicating that the value passed to quote() -** was a text value, then this function searches the input for "\n" and "\r" -** characters and adds a wrapper similar to the following: -** -** replace(replace(, '\n', char(10), '\r', char(13)); +** This function attempts to create a lost and found table within the +** output db. If successful, it returns a pointer to a buffer containing +** the name of the new table. It is the responsibility of the caller to +** eventually free this buffer using sqlite3_free(). ** -** Or, if the first character of the input is not "'", then a copy of the input -** is returned. +** If an error occurs, NULL is returned and an error code and error +** message left in the recover handle. */ -static void recoverEscapeCrlf( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static char *recoverLostAndFoundCreate( + sqlite3_recover *p, /* Recover object */ + int nField /* Number of column fields in new table */ ){ - const char *zText = (const char*)sqlite3_value_text(argv[0]); - (void)argc; - if( zText && zText[0]=='\'' ){ - int nText = sqlite3_value_bytes(argv[0]); - int i; - char zBuf1[20]; - char zBuf2[20]; - const char *zNL = 0; - const char *zCR = 0; - int nCR = 0; - int nNL = 0; + char *zTbl = 0; + sqlite3_stmt *pProbe = 0; + int ii = 0; - for(i=0; zText[i]; i++){ - if( zNL==0 && zText[i]=='\n' ){ - zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); - nNL = (int)strlen(zNL); - } - if( zCR==0 && zText[i]=='\r' ){ - zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); - nCR = (int)strlen(zCR); - } + pProbe = recoverPrepare(p, p->dbOut, + "SELECT 1 FROM sqlite_schema WHERE name=?" + ); + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ + int bFail = 0; + if( ii<0 ){ + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); + }else{ + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); } - if( zNL || zCR ){ - int iOut = 0; - i64 nMax = (nNL > nCR) ? nNL : nCR; - i64 nAlloc = nMax * nText + (nMax+64)*2; - char *zOut = (char*)sqlite3_malloc64(nAlloc); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - return; + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pProbe) ){ + bFail = 1; } + recoverReset(p, pProbe); + } - if( zNL && zCR ){ - memcpy(&zOut[iOut], "replace(replace(", 16); - iOut += 16; - }else{ - memcpy(&zOut[iOut], "replace(", 8); - iOut += 8; - } - for(i=0; zText[i]; i++){ - if( zText[i]=='\n' ){ - memcpy(&zOut[iOut], zNL, nNL); - iOut += nNL; - }else if( zText[i]=='\r' ){ - memcpy(&zOut[iOut], zCR, nCR); - iOut += nCR; - }else{ - zOut[iOut] = zText[i]; - iOut++; - } - } + if( bFail ){ + sqlite3_clear_bindings(pProbe); + sqlite3_free(zTbl); + zTbl = 0; + } + } + recoverFinalize(p, pProbe); - if( zNL ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; - memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; - } - if( zCR ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; - memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; - } + if( zTbl ){ + const char *zSep = 0; + char *zField = 0; + char *zSql = 0; - sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); - sqlite3_free(zOut); - return; + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql); + recoverSqlCallback(p, zSql); + sqlite3_free(zSql); + }else if( p->errCode==SQLITE_OK ){ + recoverError( + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound + ); } - sqlite3_result_value(context, argv[0]); + return zTbl; } /* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in -** this case. -** -** Otherwise, attempt to populate temporary table "recovery.schema" with the -** parts of the database schema that can be extracted from the input database. -** -** If no error occurs, SQLITE_OK is returned. Otherwise, an error code -** and error message are left in the recover handle and a copy of the -** error code returned. It is not considered an error if part of all of -** the database schema cannot be recovered due to corruption. +** Synthesize and prepare an INSERT statement to write to the lost_and_found +** table in the output database. The name of the table is zTab, and it has +** nField c* fields. */ -static int recoverCacheSchema(sqlite3_recover *p){ - return recoverExec(p, p->dbOut, - "WITH RECURSIVE pages(p) AS (" - " SELECT 1" - " UNION" - " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" - ")" - "INSERT INTO recovery.schema SELECT" - " max(CASE WHEN field=0 THEN value ELSE NULL END)," - " max(CASE WHEN field=1 THEN value ELSE NULL END)," - " max(CASE WHEN field=2 THEN value ELSE NULL END)," - " max(CASE WHEN field=3 THEN value ELSE NULL END)," - " max(CASE WHEN field=4 THEN value ELSE NULL END)" - "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" - " SELECT p FROM pages" - ") GROUP BY pgno, cell" - ); +static sqlite3_stmt *recoverLostAndFoundInsert( + sqlite3_recover *p, + const char *zTab, + int nField +){ + int nTotal = nField + 4; + int ii; + char *zBind = 0; + sqlite3_stmt *pRet = 0; + + if( p->xSql==0 ){ + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + }else{ + const char *zSep = ""; + for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind + ); + } + + sqlite3_free(zBind); + return pRet; } /* -** If this recover handle is not in SQL callback mode (i.e. was not created -** using sqlite3_recover_init_sql()) of if an error has already occurred, -** this function is a no-op. Otherwise, issue a callback with SQL statement -** zSql as the parameter. -** -** If the callback returns non-zero, set the recover handle error code to -** the value returned (so that the caller will abandon processing). +** Input database page iPg contains data that will be written to the +** lost-and-found table of the output database. This function attempts +** to identify the root page of the tree that page iPg belonged to. +** If successful, it sets output variable (*piRoot) to the page number +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, +** an SQLite error code is returned and the final value of *piRoot +** undefined. */ -static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ - if( p->errCode==SQLITE_OK && p->xSql ){ - int res = p->xSql(p->pSqlCtx, zSql); - if( res ){ - recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); +static int recoverLostAndFoundFindRoot( + sqlite3_recover *p, + i64 iPg, + i64 *piRoot +){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->pFindRoot==0 ){ + pLaf->pFindRoot = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE p(pgno) AS (" + " SELECT ?" + " UNION" + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" + ") " + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " + " AND m.parent IS NULL" + ); + } + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); + }else{ + *piRoot = iPg; } + recoverReset(p, pLaf->pFindRoot); } + return p->errCode; } /* -** Transfer the following settings from the input database to the output -** database: -** -** + page-size, -** + auto-vacuum settings, -** + database encoding, -** + user-version (PRAGMA user_version), and -** + application-id (PRAGMA application_id), and +** Recover data from page iPage of the input database and write it to +** the lost-and-found table in the output database. */ -static void recoverTransferSettings(sqlite3_recover *p){ - const char *aPragma[] = { - "encoding", - "page_size", - "auto_vacuum", - "user_version", - "application_id" - }; - int ii; +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_value **apVal = pLaf->apVal; + sqlite3_stmt *pPageData = pLaf->pPageData; + sqlite3_stmt *pInsert = pLaf->pInsert; - /* Truncate the output database to 0 pages in size. This is done by - ** opening a new, empty, temp db, then using the backup API to clobber - ** any existing output db with a copy of it. */ - if( p->errCode==SQLITE_OK ){ - sqlite3 *db2 = 0; - int rc = sqlite3_open("", &db2); - if( rc!=SQLITE_OK ){ - recoverDbError(p, db2); - return; - } + int nVal = -1; + int iPrevCell = 0; + i64 iRoot = 0; + int bHaveRowid = 0; + i64 iRowid = 0; + int ii = 0; - for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){ - const char *zPrag = aPragma[ii]; - sqlite3_stmt *p1 = 0; - p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag); - if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ - const char *zArg = (const char*)sqlite3_column_text(p1, 0); - char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); - recoverSqlCallback(p, z2); - recoverExec(p, db2, z2); - sqlite3_free(z2); - if( zArg==0 ){ - recoverError(p, SQLITE_NOMEM, 0); - } + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; + sqlite3_bind_int64(pPageData, 1, iPage); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ + int iCell = sqlite3_column_int64(pPageData, 0); + int iField = sqlite3_column_int64(pPageData, 1); + + if( iPrevCell!=iCell && nVal>=0 ){ + /* Insert the new row */ + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ + if( bHaveRowid ){ + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ + } + for(ii=0; iierrCode==SQLITE_OK ){ - sqlite3 *db = p->dbOut; - sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); - if( pBackup ){ - sqlite3_backup_step(pBackup, -1); - p->errCode = sqlite3_backup_finish(pBackup); - }else{ - recoverDbError(p, db); + if( iCell<0 ) break; + + if( iField<0 ){ + assert( nVal==-1 ); + iRowid = sqlite3_column_int64(pPageData, 2); + bHaveRowid = 1; + nVal = 0; + }else if( iFieldnMaxField ){ + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); + apVal[iField] = sqlite3_value_dup(pVal); + assert( iField==nVal || (nVal==-1 && iField==0) ); + nVal = iField+1; + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); } } - sqlite3_close(db2); + iPrevCell = iCell; + } + recoverReset(p, pPageData); + + for(ii=0; iierrCode!=SQLITE_OK). A copy of the error code is returned in -** this case. -** -** Otherwise, an attempt is made to open the output database, attach -** and create the schema of the temporary database used to store -** intermediate data, and to register all required user functions and -** virtual table modules with the output handle. -** -** If no error occurs, SQLITE_OK is returned. Otherwise, an error code -** and error message are left in the recover handle and a copy of the -** error code returned. -*/ -static int recoverOpenOutput(sqlite3_recover *p){ - struct Func { - const char *zName; - int nArg; - void (*xFunc)(sqlite3_context*,int,sqlite3_value **); - } aFunc[] = { - { "getpage", 1, recoverGetPage }, - { "page_is_used", 1, recoverPageIsUsed }, - { "read_i32", 2, recoverReadI32 }, - { "escape_crlf", 1, recoverEscapeCrlf }, - }; +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found +** table of the output database is populated with recovered data that can +** not be assigned to any recovered schema object. +*/ +static int recoverLostAndFound3Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + if( pLaf->pInsert==0 ){ + return SQLITE_DONE; + }else{ + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllPage); + if( res==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ + recoverLostAndFoundOnePage(p, iPage); + } + }else{ + recoverReset(p, pLaf->pAllPage); + return SQLITE_DONE; + } + } + } + } + return SQLITE_OK; +} - const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; - sqlite3 *db = 0; /* New database handle */ - int ii; /* For iterating through aFunc[] */ +/* +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 +** state - during which the lost-and-found table of the output database +** is populated with recovered data that can not be assigned to any +** recovered schema object. +*/ +static void recoverLostAndFound3Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; - assert( p->dbOut==0 ); + if( pLaf->nMaxField>0 ){ + char *zTab = 0; /* Name of lost_and_found table */ - if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ - recoverDbError(p, db); - } + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); + sqlite3_free(zTab); - /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. - ** These two are registered with the output database handle - this - ** module depends on the input handle supporting the sqlite_dbpage - ** virtual table only. */ - if( p->errCode==SQLITE_OK ){ - p->errCode = sqlite3_dbdata_init(db, 0, 0); - } + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT ii FROM seq" , p->laf.nPg + ); + pLaf->pPageData = recoverPrepare(p, p->dbOut, + "SELECT cell, field, value " + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " + "UNION ALL " + "SELECT -1, -1, -1" + ); - /* Register the custom user-functions with the output handle. */ - for(ii=0; - p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0])); - ii++){ - p->errCode = sqlite3_create_function(db, aFunc[ii].zName, - aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + pLaf->apVal = (sqlite3_value**)recoverMalloc(p, + pLaf->nMaxField*sizeof(sqlite3_value*) ); } +} + +/* +** Initialize resources required in RECOVER_STATE_WRITING state - during which +** tables recovered from the schema of the input database are populated with +** recovered data. +*/ +static int recoverWriteDataInit(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + RecoverTable *pTbl = 0; + int nByte = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + assert( p1->nMax==0 ); + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; + } + + /* Allocate an array of (sqlite3_value*) in which to accumulate the values + ** that will be written to the output database in a single row. */ + nByte = sizeof(sqlite3_value*) * (p1->nMax+1); + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); + if( p1->apVal==0 ) return p->errCode; + + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT + ** to loop through cells that appear to belong to a single table (pSel). */ + p1->pTbls = recoverPrepare(p, p->dbOut, + "SELECT rootpage FROM recovery.schema " + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" + " ORDER BY (tbl_name='sqlite_sequence') ASC" + ); + p1->pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(page) AS (" + " SELECT ?1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" + ") " + "SELECT page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0" + ); - p->dbOut = db; return p->errCode; } /* -** Attach the auxiliary database 'recovery' to the output database handle. -** This temporary database is used during the recovery process and then -** discarded. +** Clean up resources allocated by recoverWriteDataInit() (stuff in +** sqlite3_recover.w1). */ -static void recoverOpenRecovery(sqlite3_recover *p){ - char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); - recoverExec(p, p->dbOut, zSql); - recoverExec(p, p->dbOut, - "PRAGMA writable_schema = 1;" - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - ); - sqlite3_free(zSql); +static void recoverWriteDataCleanup(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + int ii; + for(ii=0; iinVal; ii++){ + sqlite3_value_free(p1->apVal[ii]); + } + sqlite3_free(p1->apVal); + recoverFinalize(p, p1->pInsert); + recoverFinalize(p, p1->pTbls); + recoverFinalize(p, p1->pSel); + memset(p1, 0, sizeof(*p1)); } - /* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). -** -** Otherwise, argument zName must be the name of a table that has just been -** created in the output database. This function queries the output db -** for the schema of said table, and creates a RecoverTable object to -** store the schema in memory. The new RecoverTable object is linked into -** the list at sqlite3_recover.pTblList. -** -** Parameter iRoot must be the root page of table zName in the INPUT -** database. -*/ -static void recoverAddTable( - sqlite3_recover *p, - const char *zName, /* Name of table created in output db */ - i64 iRoot /* Root page of same table in INPUT db */ -){ - sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, - "PRAGMA table_xinfo(%Q)", zName - ); +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_WRITING state - during which tables recovered from the +** schema of the input database are populated with recovered data. +*/ +static int recoverWriteDataStep(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + sqlite3_stmt *pSel = p1->pSel; + sqlite3_value **apVal = p1->apVal; - if( pStmt ){ - int iPk = -1; - int iBind = 1; - RecoverTable *pNew = 0; - int nCol = 0; - int nName = recoverStrlen(zName); - int nByte = 0; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - nCol++; - nByte += (sqlite3_column_bytes(pStmt, 1)+1); - } - nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; - recoverReset(p, pStmt); + if( p->errCode==SQLITE_OK && p1->pTab==0 ){ + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); + p1->pTab = recoverFindTable(p, iRoot); - pNew = recoverMalloc(p, nByte); - if( pNew ){ - int i = 0; - int iField = 0; - char *csr = 0; - pNew->aCol = (RecoverColumn*)&pNew[1]; - pNew->zTab = csr = (char*)&pNew->aCol[nCol]; - pNew->nCol = nCol; - pNew->iRoot = iRoot; - memcpy(csr, zName, nName); - csr += nName+1; + recoverFinalize(p, p1->pInsert); + p1->pInsert = 0; - for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ - int iPKF = sqlite3_column_int(pStmt, 5); - int n = sqlite3_column_bytes(pStmt, 1); - const char *z = (const char*)sqlite3_column_text(pStmt, 1); - const char *zType = (const char*)sqlite3_column_text(pStmt, 2); - int eHidden = sqlite3_column_int(pStmt, 6); + /* If this table is unknown, return early. The caller will invoke this + ** function again and it will move on to the next table. */ + if( p1->pTab==0 ) return p->errCode; - if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; - if( iPKF>1 ) iPk = -2; - pNew->aCol[i].zCol = csr; - pNew->aCol[i].eHidden = eHidden; - if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ - pNew->aCol[i].iField = -1; - }else{ - pNew->aCol[i].iField = iField++; - } - if( eHidden!=RECOVER_EHIDDEN_VIRTUAL - && eHidden!=RECOVER_EHIDDEN_STORED - ){ - pNew->aCol[i].iBind = iBind++; - } - memcpy(csr, z, n); - csr += (n+1); + /* If this is the sqlite_sequence table, delete any rows added by + ** earlier INSERT statements on tables with AUTOINCREMENT primary + ** keys before recovering its contents. The p1->pTbls SELECT statement + ** is rigged to deliver "sqlite_sequence" last of all, so we don't + ** worry about it being modified after it is recovered. */ + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); } - pNew->pNext = p->pTblList; - p->pTblList = pNew; - pNew->bIntkey = 1; + /* Bind the root page of this table within the original database to + ** SELECT statement p1->pSel. The SELECT statement will then iterate + ** through cells that look like they belong to table pTab. */ + sqlite3_bind_int64(pSel, 1, iRoot); + + p1->nVal = 0; + p1->bHaveRowid = 0; + p1->iPrevPage = -1; + p1->iPrevCell = -1; + }else{ + return SQLITE_DONE; } + } + assert( p->errCode!=SQLITE_OK || p1->pTab ); - recoverFinalize(p, pStmt); + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + RecoverTable *pTab = p1->pTab; - pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); - while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ - int iField = sqlite3_column_int(pStmt, 0); - int iCol = sqlite3_column_int(pStmt, 1); + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); - assert( iColnCol ); - pNew->aCol[iCol].iField = iField; + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); - pNew->bIntkey = 0; - iPk = -2; + if( bNewCell ){ + int ii = 0; + if( p1->nVal>=0 ){ + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ + recoverFinalize(p, p1->pInsert); + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); + p1->nInsert = p1->nVal; + } + if( p1->nVal>0 ){ + sqlite3_stmt *pInsert = p1->pInsert; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + int iBind = pCol->iBind; + if( iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, p1->iRowid); + }else if( pCol->iFieldnVal ){ + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); + } + } + } + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); + } + if( SQLITE_ROW==sqlite3_step(pInsert) ){ + const char *z = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, z); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + } + + for(ii=0; iinVal; ii++){ + sqlite3_value_free(apVal[ii]); + apVal[ii] = 0; + } + p1->nVal = -1; + p1->bHaveRowid = 0; } - recoverFinalize(p, pStmt); - if( p->errCode==SQLITE_OK ){ - if( iPk>=0 ){ - pNew->aCol[iPk].bIPK = 1; - }else if( pNew->bIntkey ){ - pNew->iRowidBind = iBind++; + if( iPage!=0 ){ + if( iField<0 ){ + p1->iRowid = sqlite3_column_int64(pSel, 3); + assert( p1->nVal==-1 ); + p1->nVal = 0; + p1->bHaveRowid = 1; + }else if( iFieldnCol ){ + assert( apVal[iField]==0 ); + apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + p1->nVal = iField+1; + }else if( pTab->nCol==0 ){ + p1->nVal = pTab->nCol; } + p1->iPrevCell = iCell; + p1->iPrevPage = iPage; } + }else{ + recoverReset(p, pSel); + p1->pTab = 0; } + + return p->errCode; } /* -** This function is called after recoverCacheSchema() has cached those parts -** of the input database schema that could be recovered in temporary table -** "recovery.schema". This function creates in the output database copies -** of all parts of that schema that must be created before the tables can -** be populated. Specifically, this means: -** -** * all tables that are not VIRTUAL, and -** * UNIQUE indexes. -** -** If the recovery handle uses SQL callbacks, then callbacks containing -** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. -** -** Additionally, records are added to the sqlite_schema table of the -** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE -** records are written directly to sqlite_schema, not actually executed. -** If the handle is in SQL callback mode, then callbacks are invoked -** with equivalent SQL statements. -*/ -static int recoverWriteSchema1(sqlite3_recover *p){ - sqlite3_stmt *pSelect = 0; - sqlite3_stmt *pTblname = 0; +** Initialize resources required by sqlite3_recover_step() in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static void recoverLostAndFound1Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_stmt *pStmt = 0; - pSelect = recoverPrepare(p, p->dbOut, - "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" - " SELECT rootpage, name, sql, " - " type='table', " - " sql LIKE 'create virtual%'," - " (type='index' AND (sql LIKE '%unique%' OR ?1))" - " FROM recovery.schema" - ")" - "SELECT rootpage, tbl, isVirtual, name, sql" - " FROM dbschema " - " WHERE tbl OR isIndex" - " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" - ); + assert( p->laf.pUsed==0 ); + pLaf->nPg = recoverPageCount(p); + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); - pTblname = recoverPrepare(p, p->dbOut, - "SELECT name FROM sqlite_schema " - "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + /* Prepare a statement to iterate through all pages that are part of any tree + ** in the recoverable part of the input database schema to the bitmap. And, + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the + ** freelist. */ + pStmt = recoverPrepare( + p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" + " SELECT 1 UNION ALL" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); + pLaf->pUsedPages = pStmt; +} - if( pSelect ){ - sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); - while( sqlite3_step(pSelect)==SQLITE_ROW ){ - i64 iRoot = sqlite3_column_int64(pSelect, 0); - int bTable = sqlite3_column_int(pSelect, 1); - int bVirtual = sqlite3_column_int(pSelect, 2); - const char *zName = (const char*)sqlite3_column_text(pSelect, 3); - const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); - char *zFree = 0; - int rc = SQLITE_OK; - - if( bVirtual ){ - zSql = (const char*)(zFree = recoverMPrintf(p, - "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", - zName, zName, zSql - )); - } - rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); - if( rc==SQLITE_OK ){ - recoverSqlCallback(p, zSql); - if( bTable && !bVirtual ){ - if( SQLITE_ROW==sqlite3_step(pTblname) ){ - const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - if( zTbl ) recoverAddTable(p, zTbl, iRoot); - } - recoverReset(p, pTblname); - } - }else if( rc!=SQLITE_ERROR ){ - recoverDbError(p, p->dbOut); - } - sqlite3_free(zFree); +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static int recoverLostAndFound1Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + int rc = p->errCode; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pLaf->pUsedPages); + if( rc==SQLITE_ROW ){ + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); + recoverBitmapSet(pLaf->pUsed, iPg); + rc = SQLITE_OK; + }else{ + recoverFinalize(p, pLaf->pUsedPages); + pLaf->pUsedPages = 0; } } - recoverFinalize(p, pSelect); - recoverFinalize(p, pTblname); - - return p->errCode; + return rc; } /* -** This function is called after the output database has been populated. It -** adds all recovered schema elements that were not created in the output -** database by recoverWriteSchema1() - everything except for tables and -** UNIQUE indexes. Specifically: -** -** * views, -** * triggers, -** * non-UNIQUE indexes. -** -** If the recover handle is in SQL callback mode, then equivalent callbacks -** are issued to create the schema elements. -*/ -static int recoverWriteSchema2(sqlite3_recover *p){ - sqlite3_stmt *pSelect = 0; +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 +** are sorted into sets that likely belonged to the same database tree. +*/ +static void recoverLostAndFound2Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; - pSelect = recoverPrepare(p, p->dbOut, - p->bSlowIndexes ? - "SELECT rootpage, sql FROM recovery.schema " - " WHERE type!='table' AND type!='index'" - : - "SELECT rootpage, sql FROM recovery.schema " - " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + assert( p->laf.pAllAndParent==0 ); + assert( p->laf.pMapInsert==0 ); + assert( p->laf.pMaxField==0 ); + assert( p->laf.nMaxField==0 ); + + pLaf->pMapInsert = recoverPrepare(p, p->dbOut, + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" + ); + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT pgno, child FROM sqlite_dbptr('getpage()') " + " UNION ALL " + "SELECT NULL, ii FROM seq", p->laf.nPg ); + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" + ); +} - if( pSelect ){ - while( sqlite3_step(pSelect)==SQLITE_ROW ){ - const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); - int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); - if( rc==SQLITE_OK ){ - recoverSqlCallback(p, zSql); - }else if( rc!=SQLITE_ERROR ){ - recoverDbError(p, p->dbOut); +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged +** to the same database tree. +*/ +static int recoverLostAndFound2Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllAndParent); + if( res==SQLITE_ROW ){ + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); + sqlite3_bind_value(pLaf->pMapInsert, 2, + sqlite3_column_value(pLaf->pAllAndParent, 0) + ); + sqlite3_step(pLaf->pMapInsert); + recoverReset(p, pLaf->pMapInsert); + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ + int nMax = sqlite3_column_int(pLaf->pMaxField, 0); + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; + } + recoverReset(p, pLaf->pMaxField); } + }else{ + recoverFinalize(p, pLaf->pAllAndParent); + pLaf->pAllAndParent =0; + return SQLITE_DONE; } } - recoverFinalize(p, pSelect); - return p->errCode; } /* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). In this case it returns NULL. -** -** Otherwise, if the recover handle is configured to create an output -** database (was created by sqlite3_recover_init()), then this function -** prepares and returns an SQL statement to INSERT a new record into table -** pTab, assuming the first nField fields of a record extracted from disk -** are valid. -** -** For example, if table pTab is: -** -** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); -** -** And nField is 4, then the SQL statement prepared and returned is: -** -** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); -** -** In this case even though 4 values were extracted from the input db, -** only 3 are written to the output, as the generated STORED column -** cannot be written. -** -** If the recover handle is in SQL callback mode, then the SQL statement -** prepared is such that evaluating it returns a single row containing -** a single text value - itself an SQL statement similar to the above, -** except with SQL literals in place of the variables. For example: -** -** SELECT 'INSERT INTO (a, c, d) VALUES (' -** || quote(?1) || ', ' -** || quote(?2) || ', ' -** || quote(?3) || ')'; -** -** In either case, it is the responsibility of the caller to eventually -** free the statement handle using sqlite3_finalize(). +** Free all resources allocated as part of sqlite3_recover_step() calls +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. */ -static sqlite3_stmt *recoverInsertStmt( - sqlite3_recover *p, - RecoverTable *pTab, - int nField -){ - sqlite3_stmt *pRet = 0; - const char *zSep = ""; - const char *zSqlSep = ""; - char *zSql = 0; - char *zFinal = 0; - char *zBind = 0; - int ii; - int bSql = p->xSql ? 1 : 0; - - if( nField<=0 ) return 0; - - assert( nField<=pTab->nCol ); - - zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); +static void recoverLostAndFoundCleanup(sqlite3_recover *p){ + recoverBitmapFree(p->laf.pUsed); + p->laf.pUsed = 0; + sqlite3_finalize(p->laf.pUsedPages); + sqlite3_finalize(p->laf.pAllAndParent); + sqlite3_finalize(p->laf.pMapInsert); + sqlite3_finalize(p->laf.pMaxField); + sqlite3_finalize(p->laf.pFindRoot); + sqlite3_finalize(p->laf.pInsert); + sqlite3_finalize(p->laf.pAllPage); + sqlite3_finalize(p->laf.pPageData); + p->laf.pUsedPages = 0; + p->laf.pAllAndParent = 0; + p->laf.pMapInsert = 0; + p->laf.pMaxField = 0; + p->laf.pFindRoot = 0; + p->laf.pInsert = 0; + p->laf.pAllPage = 0; + p->laf.pPageData = 0; + sqlite3_free(p->laf.apVal); + p->laf.apVal = 0; +} - if( pTab->iRowidBind ){ - assert( pTab->bIntkey ); - zSql = recoverMPrintf(p, "%z_rowid_", zSql); - if( bSql ){ - zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); - }else{ - zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); - } - zSqlSep = "||', '||"; - zSep = ", "; - } +/* +** Free all resources allocated as part of sqlite3_recover_step() calls. +*/ +static void recoverFinalCleanup(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; - for(ii=0; iiaCol[ii].eHidden; - if( eHidden!=RECOVER_EHIDDEN_VIRTUAL - && eHidden!=RECOVER_EHIDDEN_STORED - ){ - assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); - zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + recoverWriteDataCleanup(p); + recoverLostAndFoundCleanup(p); - if( bSql ){ - zBind = recoverMPrintf(p, - "%z%sescape_crlf(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind - ); - zSqlSep = "||', '||"; - }else{ - zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); - } - zSep = ", "; - } + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); - if( bSql ){ - zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", - zSql, zBind - ); - }else{ - zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); + { +#ifndef NDEBUG + int res = +#endif + sqlite3_close(p->dbOut); + assert( res==SQLITE_OK ); } + p->dbOut = 0; +} - pRet = recoverPrepare(p, p->dbOut, zFinal); - sqlite3_free(zSql); - sqlite3_free(zBind); - sqlite3_free(zFinal); - - return pRet; +/* +** Decode and return an unsigned 16-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); } +/* +** Decode and return an unsigned 32-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} /* -** Search the list of RecoverTable objects at p->pTblList for one that -** has root page iRoot in the input database. If such an object is found, -** return a pointer to it. Otherwise, return NULL. +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) +** and return the number of bytes consumed. */ -static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ - RecoverTable *pRet = 0; - for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); - return pRet; +static int recoverGetVarint(const u8 *a, i64 *pVal){ + sqlite3_uint64 u = 0; + int i; + for(i=0; i<8; i++){ + u = (u<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } + } + u = (u<<8) + (a[i]&0xff); + *pVal = (sqlite3_int64)u; + return 9; } /* -** This function attempts to create a lost and found table within the -** output db. If successful, it returns a pointer to a buffer containing -** the name of the new table. It is the responsibility of the caller to -** eventually free this buffer using sqlite3_free(). -** -** If an error occurs, NULL is returned and an error code and error -** message left in the recover handle. +** The second argument points to a buffer n bytes in size. If this buffer +** or a prefix thereof appears to contain a well-formed SQLite b-tree page, +** return the page-size in bytes. Otherwise, if the buffer does not +** appear to contain a well-formed b-tree page, return 0. */ -static char *recoverLostAndFoundCreate( - sqlite3_recover *p, /* Recover object */ - int nField /* Number of column fields in new table */ -){ - char *zTbl = 0; - sqlite3_stmt *pProbe = 0; +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ + u8 *aUsed = aTmp; + int nFrag = 0; + int nActual = 0; + int iFree = 0; + int nCell = 0; /* Number of cells on page */ + int iCellOff = 0; /* Offset of cell array in page */ + int iContent = 0; + int eType = 0; int ii = 0; - pProbe = recoverPrepare(p, p->dbOut, - "SELECT 1 FROM sqlite_schema WHERE name=?" - ); - for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ - int bFail = 0; - if( ii<0 ){ - zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); - }else{ - zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); + eType = (int)a[0]; + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; + + iFree = (int)recoverGetU16(&a[1]); + nCell = (int)recoverGetU16(&a[3]); + iContent = (int)recoverGetU16(&a[5]); + if( iContent==0 ) iContent = 65536; + nFrag = (int)a[7]; + + if( iContent>n ) return 0; + + memset(aUsed, 0, n); + memset(aUsed, 0xFF, iContent); + + /* Follow the free-list. This is the same format for all b-tree pages. */ + if( iFree && iFree<=iContent ) return 0; + while( iFree ){ + int iNext = 0; + int nByte = 0; + if( iFree>(n-4) ) return 0; + iNext = recoverGetU16(&a[iFree]); + nByte = recoverGetU16(&a[iFree+2]); + if( iFree+nByte>n || nByte<4 ) return 0; + if( iNext && iNextiContent ) return 0; + for(ii=0; iin ){ + return 0; + } + if( eType==0x05 || eType==0x02 ) nByte += 4; + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); + if( eType==0x0D ){ + i64 dummy = 0; + nByte += recoverGetVarint(&a[iOff+nByte], &dummy); } + if( eType!=0x05 ){ + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); + int M = ((n-12)*32/255)-23; + int K = M+((nPayload-M)%(n-4)); - if( p->errCode==SQLITE_OK ){ - sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pProbe) ){ - bFail = 1; + if( nPayloadn ){ + return 0; + } + for(iByte=iOff; iByte<(iOff+nByte); iByte++){ + if( aUsed[iByte]!=0 ){ + return 0; + } + aUsed[iByte] = 0xFF; } } - recoverFinalize(p, pProbe); - if( zTbl ){ - const char *zSep = 0; - char *zField = 0; - char *zSql = 0; + nActual = 0; + for(ii=0; iierrCode==SQLITE_OK && iidbOut, zSql); - recoverSqlCallback(p, zSql); - sqlite3_free(zSql); - }else if( p->errCode==SQLITE_OK ){ - recoverError( - p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound - ); - } +static sqlite3_io_methods recover_methods = { + 2, /* iVersion */ + recoverVfsClose, + recoverVfsRead, + recoverVfsWrite, + recoverVfsTruncate, + recoverVfsSync, + recoverVfsFileSize, + recoverVfsLock, + recoverVfsUnlock, + recoverVfsCheckReservedLock, + recoverVfsFileControl, + recoverVfsSectorSize, + recoverVfsDeviceCharacteristics, + recoverVfsShmMap, + recoverVfsShmLock, + recoverVfsShmBarrier, + recoverVfsShmUnmap, + recoverVfsFetch, + recoverVfsUnfetch +}; - return zTbl; +static int recoverVfsClose(sqlite3_file *pFd){ + assert( pFd->pMethods!=&recover_methods ); + return pFd->pMethods->xClose(pFd); } /* -** Synthesize and prepare an INSERT statement to write to the lost_and_found -** table in the output database. The name of the table is zTab, and it has -** nField c* fields. +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. */ -static sqlite3_stmt *recoverLostAndFoundInsert( - sqlite3_recover *p, - const char *zTab, - int nField -){ - int nTotal = nField + 4; - int ii; - char *zBind = 0; - sqlite3_stmt *pRet = 0; - - if( p->xSql==0 ){ - for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind - ); - }else{ - const char *zSep = ""; - for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind - ); - } +static void recoverPutU16(u8 *a, u32 v){ + a[0] = (v>>8) & 0x00FF; + a[1] = (v>>0) & 0x00FF; +} - sqlite3_free(zBind); - return pRet; +/* +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +*/ +static void recoverPutU32(u8 *a, u32 v){ + a[0] = (v>>24) & 0x00FF; + a[1] = (v>>16) & 0x00FF; + a[2] = (v>>8) & 0x00FF; + a[3] = (v>>0) & 0x00FF; } /* -** Input database page iPg contains data that will be written to the -** lost-and-found table of the output database. This function attempts -** to identify the root page of the tree that page iPg belonged to. -** If successful, it sets output variable (*piRoot) to the page number -** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, -** an SQLite error code is returned and the final value of *piRoot -** undefined. +** Detect the page-size of the database opened by file-handle pFd by +** searching the first part of the file for a well-formed SQLite b-tree +** page. If parameter nReserve is non-zero, then as well as searching for +** a b-tree page with zero reserved bytes, this function searches for one +** with nReserve reserved bytes at the end of it. +** +** If successful, set variable p->detected_pgsz to the detected page-size +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code +** is returned. The final value of p->detected_pgsz is undefined in this +** case. */ -static int recoverLostAndFoundFindRoot( - sqlite3_recover *p, - i64 iPg, - i64 *piRoot +static int recoverVfsDetectPagesize( + sqlite3_recover *p, /* Recover handle */ + sqlite3_file *pFd, /* File-handle open on input database */ + u32 nReserve, /* Possible nReserve value */ + i64 nSz /* Size of database file in bytes */ ){ - RecoverStateLAF *pLaf = &p->laf; + int rc = SQLITE_OK; + const int nMin = 512; + const int nMax = 65536; + const int nMaxBlk = 4; + u32 pgsz = 0; + int iBlk = 0; + u8 *aPg = 0; + u8 *aTmp = 0; + int nBlk = 0; - if( pLaf->pFindRoot==0 ){ - pLaf->pFindRoot = recoverPrepare(p, p->dbOut, - "WITH RECURSIVE p(pgno) AS (" - " SELECT ?" - " UNION" - " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" - ") " - "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " - " AND m.parent IS NULL" - ); - } - if( p->errCode==SQLITE_OK ){ - sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); - if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ - *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); - }else{ - *piRoot = iPg; + aPg = (u8*)sqlite3_malloc(2*nMax); + if( aPg==0 ) return SQLITE_NOMEM; + aTmp = &aPg[nMax]; + + nBlk = (nSz+nMax-1)/nMax; + if( nBlk>nMaxBlk ) nBlk = nMaxBlk; + + do { + for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOffpFindRoot); - } - return p->errCode; + if( pgsz>(u32)p->detected_pgsz ){ + p->detected_pgsz = pgsz; + p->nReserve = nReserve; + } + if( nReserve==0 ) break; + nReserve = 0; + }while( 1 ); + + p->detected_pgsz = pgsz; + sqlite3_free(aPg); + return rc; } /* -** Recover data from page iPage of the input database and write it to -** the lost-and-found table in the output database. +** The xRead() method of the wrapper VFS. This is used to intercept calls +** to read page 1 of the input database. */ -static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ - RecoverStateLAF *pLaf = &p->laf; - sqlite3_value **apVal = pLaf->apVal; - sqlite3_stmt *pPageData = pLaf->pPageData; - sqlite3_stmt *pInsert = pLaf->pInsert; +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ + int rc = SQLITE_OK; + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + if( nByte==16 ){ + sqlite3_randomness(16, aBuf); + }else + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ + /* Ensure that the database has a valid header file. The only fields + ** that really matter to recovery are: + ** + ** + Database page size (16-bits at offset 16) + ** + Size of db in pages (32-bits at offset 28) + ** + Database encoding (32-bits at offset 56) + ** + ** Also preserved are: + ** + ** + first freelist page (32-bits at offset 32) + ** + size of freelist (32-bits at offset 36) + ** + the wal-mode flags (16-bits at offset 18) + ** + ** We also try to preserve the auto-vacuum, incr-value, user-version + ** and application-id fields - all 32 bit quantities at offsets + ** 52, 60, 64 and 68. All other fields are set to known good values. + ** + ** Byte offset 105 should also contain the page-size as a 16-bit + ** integer. + */ + const int aPreserve[] = {32, 36, 52, 60, 64, 68}; + u8 aHdr[108] = { + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x5b, 0x30, - int nVal = -1; - int iPrevCell = 0; - i64 iRoot = 0; - int bHaveRowid = 0; - i64 iRowid = 0; - int ii = 0; + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 + }; + u8 *a = (u8*)aBuf; - if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; - sqlite3_bind_int64(pPageData, 1, iPage); - while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ - int iCell = sqlite3_column_int64(pPageData, 0); - int iField = sqlite3_column_int64(pPageData, 1); + u32 pgsz = recoverGetU16(&a[16]); + u32 nReserve = a[20]; + u32 enc = recoverGetU32(&a[56]); + u32 dbsz = 0; + i64 dbFileSize = 0; + int ii; + sqlite3_recover *p = recover_g.p; - if( iPrevCell!=iCell && nVal>=0 ){ - /* Insert the new row */ - sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ - sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ - sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ - if( bHaveRowid ){ - sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ - } - for(ii=0; iipMethods->xFileSize(pFd, &dbFileSize); + + if( rc==SQLITE_OK && p->detected_pgsz==0 ){ + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); } - if( sqlite3_step(pInsert)==SQLITE_ROW ){ - recoverSqlCallback(p, (const char*)sqlite3_column_text(pInsert, 0)); + if( p->detected_pgsz ){ + pgsz = p->detected_pgsz; + nReserve = p->nReserve; } - recoverReset(p, pInsert); - /* Discard the accumulated row data */ - for(ii=0; iinMaxField ){ - sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); - apVal[iField] = sqlite3_value_dup(pVal); - assert( iField==nVal || (nVal==-1 && iField==0) ); - nVal = iField+1; - if( apVal[iField]==0 ){ - recoverError(p, SQLITE_NOMEM, 0); + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ + enc = SQLITE_UTF8; } - } - - iPrevCell = iCell; - } - recoverReset(p, pPageData); - for(ii=0; iipPage1Cache); + p->pPage1Cache = 0; + p->pPage1Disk = 0; -/* -** Perform one step (sqlite3_recover_step()) of work for the connection -** passed as the only argument, which is guaranteed to be in -** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found -** table of the output database is populated with recovered data that can -** not be assigned to any recovered schema object. -*/ -static int recoverLostAndFound3Step(sqlite3_recover *p){ - RecoverStateLAF *pLaf = &p->laf; - if( p->errCode==SQLITE_OK ){ - if( pLaf->pInsert==0 ){ - return SQLITE_DONE; - }else{ - if( p->errCode==SQLITE_OK ){ - int res = sqlite3_step(pLaf->pAllPage); - if( res==SQLITE_ROW ){ - i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); - if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ - recoverLostAndFoundOnePage(p, iPage); - } - }else{ - recoverReset(p, pLaf->pAllPage); - return SQLITE_DONE; + p->pgsz = nByte; + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); + if( p->pPage1Cache ){ + p->pPage1Disk = &p->pPage1Cache[nByte]; + memcpy(p->pPage1Disk, aBuf, nByte); + aHdr[18] = a[18]; + aHdr[19] = a[19]; + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + recoverPutU16(&aHdr[105], pgsz-nReserve); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + aHdr[20] = nReserve; + for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){ + memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4); } + memcpy(aBuf, aHdr, sizeof(aHdr)); + memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr)); + + memcpy(p->pPage1Cache, aBuf, nByte); + }else{ + rc = p->errCode; } + } + pFd->pMethods = &recover_methods; + }else{ + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); } - return SQLITE_OK; + return rc; } /* -** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 -** state - during which the lost-and-found table of the output database -** is populated with recovered data that can not be assigned to any -** recovered schema object. -*/ -static void recoverLostAndFound3Init(sqlite3_recover *p){ - RecoverStateLAF *pLaf = &p->laf; - - if( pLaf->nMaxField>0 ){ - char *zTab = 0; /* Name of lost_and_found table */ - - zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); - pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); - sqlite3_free(zTab); - - pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, - "WITH RECURSIVE seq(ii) AS (" - " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" - ")" - "SELECT ii FROM seq" , p->laf.nPg - ); - pLaf->pPageData = recoverPrepare(p, p->dbOut, - "SELECT cell, field, value " - "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " - "UNION ALL " - "SELECT -1, -1, -1" - ); +** Used to make sqlite3_io_methods wrapper methods less verbose. +*/ +#define RECOVER_VFS_WRAPPER(code) \ + int rc = SQLITE_OK; \ + if( pFd->pMethods==&recover_methods ){ \ + pFd->pMethods = recover_g.pMethods; \ + rc = code; \ + pFd->pMethods = &recover_methods; \ + }else{ \ + rc = code; \ + } \ + return rc; - pLaf->apVal = (sqlite3_value**)recoverMalloc(p, - pLaf->nMaxField*sizeof(sqlite3_value*) - ); +/* +** Methods of the wrapper VFS. All methods except for xRead() and xClose() +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent +** method on the lower level VFS, then reinstall the wrapper before returning. +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +*/ +static int recoverVfsWrite( + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) + ); +} +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xTruncate(pFd, size) + ); +} +static int recoverVfsSync(sqlite3_file *pFd, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSync(pFd, flags) + ); +} +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileSize(pFd, pSize) + ); +} +static int recoverVfsLock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xLock(pFd, eLock) + ); +} +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xUnlock(pFd, eLock) + ); +} +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xCheckReservedLock(pFd, pResOut) + ); +} +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ + RECOVER_VFS_WRAPPER ( + (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) + ); +} +static int recoverVfsSectorSize(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSectorSize(pFd) + ); +} +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xDeviceCharacteristics(pFd) + ); +} +static int recoverVfsShmMap( + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) + ); +} +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmLock(pFd, offset, n, flags) + ); +} +static void recoverVfsShmBarrier(sqlite3_file *pFd){ + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + pFd->pMethods->xShmBarrier(pFd); + pFd->pMethods = &recover_methods; + }else{ + pFd->pMethods->xShmBarrier(pFd); } } +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmUnmap(pFd, deleteFlag) + ); +} -/* -** Initialize resources required in RECOVER_STATE_WRITING state - during which -** tables recovered from the schema of the input database are populated with -** recovered data. -*/ -static int recoverWriteDataInit(sqlite3_recover *p){ - RecoverStateW1 *p1 = &p->w1; - RecoverTable *pTbl = 0; - int nByte = 0; +static int recoverVfsFetch( + sqlite3_file *pFd, + sqlite3_int64 iOff, + int iAmt, + void **pp +){ + (void)pFd; + (void)iOff; + (void)iAmt; + *pp = 0; + return SQLITE_OK; +} +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ + (void)pFd; + (void)iOff; + (void)p; + return SQLITE_OK; +} - /* Figure out the maximum number of columns for any table in the schema */ - assert( p1->nMax==0 ); - for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ - if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; +/* +** Install the VFS wrapper around the file-descriptor open on the input +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held +** when this function is called. +*/ +static void recoverInstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + assert( recover_g.pMethods==0 ); + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + assert( pFd==0 || pFd->pMethods!=&recover_methods ); + if( pFd && pFd->pMethods ){ + int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); + recover_g.pMethods = pFd->pMethods; + recover_g.p = p; + recover_methods.iVersion = iVersion; + pFd->pMethods = &recover_methods; } - - /* Allocate an array of (sqlite3_value*) in which to accumulate the values - ** that will be written to the output database in a single row. */ - nByte = sizeof(sqlite3_value*) * (p1->nMax+1); - p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); - if( p1->apVal==0 ) return p->errCode; - - /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT - ** to loop through cells that appear to belong to a single table (pSel). */ - p1->pTbls = recoverPrepare(p, p->dbOut, - "SELECT rootpage FROM recovery.schema " - " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" - " ORDER BY (tbl_name='sqlite_sequence') ASC" - ); - p1->pSel = recoverPrepare(p, p->dbOut, - "WITH RECURSIVE pages(page) AS (" - " SELECT ?1" - " UNION" - " SELECT child FROM sqlite_dbptr('getpage()'), pages " - " WHERE pgno=page" - ") " - "SELECT page, cell, field, value " - "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " - "UNION ALL " - "SELECT 0, 0, 0, 0" - ); - - return p->errCode; } /* -** Clean up resources allocated by recoverWriteDataInit() (stuff in -** sqlite3_recover.w1). +** Uninstall the VFS wrapper that was installed around the file-descriptor open +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be +** held when this function is called. */ -static void recoverWriteDataCleanup(sqlite3_recover *p){ - RecoverStateW1 *p1 = &p->w1; - int ii; - for(ii=0; iinVal; ii++){ - sqlite3_value_free(p1->apVal[ii]); +static void recoverUninstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); + if( pFd && pFd->pMethods ){ + pFd->pMethods = recover_g.pMethods; + recover_g.pMethods = 0; + recover_g.p = 0; } - sqlite3_free(p1->apVal); - recoverFinalize(p, p1->pInsert); - recoverFinalize(p, p1->pTbls); - recoverFinalize(p, p1->pSel); - memset(p1, 0, sizeof(*p1)); } /* -** Perform one step (sqlite3_recover_step()) of work for the connection -** passed as the only argument, which is guaranteed to be in -** RECOVER_STATE_WRITING state - during which tables recovered from the -** schema of the input database are populated with recovered data. -*/ -static int recoverWriteDataStep(sqlite3_recover *p){ - RecoverStateW1 *p1 = &p->w1; - sqlite3_stmt *pSel = p1->pSel; - sqlite3_value **apVal = p1->apVal; +** This function does the work of a single sqlite3_recover_step() call. It +** is guaranteed that the handle is not in an error state when this +** function is called. +*/ +static void recoverStep(sqlite3_recover *p){ + assert( p && p->errCode==SQLITE_OK ); + switch( p->eState ){ + case RECOVER_STATE_INIT: { + int bUseWrapper = 1; + /* This is the very first call to sqlite3_recover_step() on this object. + */ + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + recoverSqlCallback(p, "PRAGMA foreign_keys = off"); + + recoverEnterMutex(); - if( p->errCode==SQLITE_OK && p1->pTab==0 ){ - if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ - i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); - p1->pTab = recoverFindTable(p, iRoot); + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); - recoverFinalize(p, p1->pInsert); - p1->pInsert = 0; + /* Attempt to open a transaction and read page 1 of the input database. + ** Two attempts may be made - one with a wrapper installed to ensure + ** that the database header is sane, and then if that attempt returns + ** SQLITE_NOTADB, then again with no wrapper. The second attempt is + ** required for encrypted databases. */ + if( p->errCode==SQLITE_OK ){ + do{ + p->errCode = SQLITE_OK; + if( bUseWrapper ) recoverInstallWrapper(p); - /* If this table is unknown, return early. The caller will invoke this - ** function again and it will move on to the next table. */ - if( p1->pTab==0 ) return p->errCode; + /* Open a transaction on the input database. */ + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); + recoverExec(p, p->dbIn, "BEGIN"); + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); + recoverTransferSettings(p); + recoverOpenRecovery(p); + recoverCacheSchema(p); - /* If this is the sqlite_sequence table, delete any rows added by - ** earlier INSERT statements on tables with AUTOINCREMENT primary - ** keys before recovering its contents. The p1->pTbls SELECT statement - ** is rigged to deliver "sqlite_sequence" last of all, so we don't - ** worry about it being modified after it is recovered. */ - if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ - recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); - recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + if( bUseWrapper ) recoverUninstallWrapper(p); + }while( p->errCode==SQLITE_NOTADB + && (bUseWrapper--) + && SQLITE_OK==recoverOneStmt(p->dbIn, "ROLLBACK") + ); } - /* Bind the root page of this table within the original database to - ** SELECT statement p1->pSel. The SELECT statement will then iterate - ** through cells that look like they belong to table pTab. */ - sqlite3_bind_int64(pSel, 1, iRoot); - - p1->nVal = 0; - p1->bHaveRowid = 0; - p1->iPrevPage = -1; - p1->iPrevCell = -1; - }else{ - return SQLITE_DONE; + recoverLeaveMutex(); + recoverExec(p, p->dbOut, "BEGIN"); + recoverWriteSchema1(p); + p->eState = RECOVER_STATE_WRITING; + break; } - } - assert( p->errCode!=SQLITE_OK || p1->pTab ); - - if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ - RecoverTable *pTab = p1->pTab; - - i64 iPage = sqlite3_column_int64(pSel, 0); - int iCell = sqlite3_column_int(pSel, 1); - int iField = sqlite3_column_int(pSel, 2); - sqlite3_value *pVal = sqlite3_column_value(pSel, 3); - int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); - - assert( bNewCell==0 || (iField==-1 || iField==0) ); - assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); - - if( bNewCell ){ - int ii = 0; - if( p1->nVal>=0 ){ - if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ - recoverFinalize(p, p1->pInsert); - p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); - p1->nInsert = p1->nVal; - } - if( p1->nVal>0 ){ - sqlite3_stmt *pInsert = p1->pInsert; - for(ii=0; iinCol; ii++){ - RecoverColumn *pCol = &pTab->aCol[ii]; - int iBind = pCol->iBind; - if( iBind>0 ){ - if( pCol->bIPK ){ - sqlite3_bind_int64(pInsert, iBind, p1->iRowid); - }else if( pCol->iFieldnVal ){ - recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); - } - } - } - if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ - sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); - } - if( SQLITE_ROW==sqlite3_step(pInsert) ){ - const char *z = (const char*)sqlite3_column_text(pInsert, 0); - recoverSqlCallback(p, z); - } - recoverReset(p, pInsert); - assert( p->errCode || pInsert ); - if( pInsert ) sqlite3_clear_bindings(pInsert); + + case RECOVER_STATE_WRITING: { + if( p->w1.pTbls==0 ){ + recoverWriteDataInit(p); + } + if( SQLITE_DONE==recoverWriteDataStep(p) ){ + recoverWriteDataCleanup(p); + if( p->zLostAndFound ){ + p->eState = RECOVER_STATE_LOSTANDFOUND1; + }else{ + p->eState = RECOVER_STATE_SCHEMA2; } } + break; + } - for(ii=0; iinVal; ii++){ - sqlite3_value_free(apVal[ii]); - apVal[ii] = 0; + case RECOVER_STATE_LOSTANDFOUND1: { + if( p->laf.pUsed==0 ){ + recoverLostAndFound1Init(p); } - p1->nVal = -1; - p1->bHaveRowid = 0; + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND2; + } + break; + } + case RECOVER_STATE_LOSTANDFOUND2: { + if( p->laf.pAllAndParent==0 ){ + recoverLostAndFound2Init(p); + } + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND3; + } + break; } - if( iPage!=0 ){ - if( iField<0 ){ - p1->iRowid = sqlite3_column_int64(pSel, 3); - assert( p1->nVal==-1 ); - p1->nVal = 0; - p1->bHaveRowid = 1; - }else if( iFieldnCol ){ - assert( apVal[iField]==0 ); - apVal[iField] = sqlite3_value_dup( pVal ); - if( apVal[iField]==0 ){ - recoverError(p, SQLITE_NOMEM, 0); - } - p1->nVal = iField+1; - }else if( pTab->nCol==0 ){ - p1->nVal = pTab->nCol; + case RECOVER_STATE_LOSTANDFOUND3: { + if( p->laf.pInsert==0 ){ + recoverLostAndFound3Init(p); } - p1->iPrevCell = iCell; - p1->iPrevPage = iPage; + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ + p->eState = RECOVER_STATE_SCHEMA2; + } + break; } - }else{ - recoverReset(p, pSel); - p1->pTab = 0; - } - return p->errCode; -} + case RECOVER_STATE_SCHEMA2: { + int rc = SQLITE_OK; -/* -** Initialize resources required by sqlite3_recover_step() in -** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not -** already allocated to a recovered schema element is determined. -*/ -static void recoverLostAndFound1Init(sqlite3_recover *p){ - RecoverStateLAF *pLaf = &p->laf; - sqlite3_stmt *pStmt = 0; + recoverWriteSchema2(p); + p->eState = RECOVER_STATE_DONE; - assert( p->laf.pUsed==0 ); - pLaf->nPg = recoverPageCount(p); - pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); + /* If no error has occurred, commit the write transaction on the output + ** database. Regardless of whether or not an error has occurred, make + ** an attempt to end the read transaction on the input database. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = recoverOneStmt(p->dbIn, "END"); + if( p->errCode==SQLITE_OK ) p->errCode = rc; - /* Prepare a statement to iterate through all pages that are part of any tree - ** in the recoverable part of the input database schema to the bitmap. And, - ** if !p->bFreelistCorrupt, add all pages that appear to be part of the - ** freelist. */ - pStmt = recoverPrepare( - p, p->dbOut, - "WITH trunk(pgno) AS (" - " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" - " UNION" - " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" - ")," - "trunkdata(pgno, data) AS (" - " SELECT pgno, getpage(pgno) FROM trunk" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" - " UNION ALL" - " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" - ")," - "" - "roots(r) AS (" - " SELECT 1 UNION ALL" - " SELECT rootpage FROM recovery.schema WHERE rootpage>0" - ")," - "used(page) AS (" - " SELECT r FROM roots" - " UNION" - " SELECT child FROM sqlite_dbptr('getpage()'), used " - " WHERE pgno=page" - ") " - "SELECT page FROM used" - " UNION ALL " - "SELECT freepgno FROM freelist WHERE NOT ?" - ); - if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); - pLaf->pUsedPages = pStmt; + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + p->eState = RECOVER_STATE_DONE; + recoverFinalCleanup(p); + break; + }; + + case RECOVER_STATE_DONE: { + /* no-op */ + break; + }; + } } + /* -** Perform one step (sqlite3_recover_step()) of work for the connection -** passed as the only argument, which is guaranteed to be in -** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not -** already allocated to a recovered schema element is determined. -*/ -static int recoverLostAndFound1Step(sqlite3_recover *p){ - RecoverStateLAF *pLaf = &p->laf; - int rc = p->errCode; - if( rc==SQLITE_OK ){ - rc = sqlite3_step(pLaf->pUsedPages); - if( rc==SQLITE_ROW ){ - i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); - recoverBitmapSet(pLaf->pUsed, iPg); - rc = SQLITE_OK; - }else{ - recoverFinalize(p, pLaf->pUsedPages); - pLaf->pUsedPages = 0; - } +** This is a worker function that does the heavy lifting for both init +** functions: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** All this function does is allocate space for the recover handle and +** take copies of the input parameters. All the real work is done within +** sqlite3_recover_run(). +*/ +sqlite3_recover *recoverInit( + sqlite3* db, + const char *zDb, + const char *zUri, /* Output URI for _recover_init() */ + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ + void *pSqlCtx /* Context arg for _recover_init_sql() */ +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); + pRet->xSql = xSql; + pRet->pSqlCtx = pSqlCtx; + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; } - return rc; + + return pRet; } /* -** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 -** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 -** are sorted into sets that likely belonged to the same database tree. -*/ -static void recoverLostAndFound2Init(sqlite3_recover *p){ - RecoverStateLAF *pLaf = &p->laf; - - assert( p->laf.pAllAndParent==0 ); - assert( p->laf.pMapInsert==0 ); - assert( p->laf.pMaxField==0 ); - assert( p->laf.nMaxField==0 ); +** Initialize a recovery handle that creates a new database containing +** the recovered data. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + return recoverInit(db, zDb, zUri, 0, 0); +} - pLaf->pMapInsert = recoverPrepare(p, p->dbOut, - "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" - ); - pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, - "WITH RECURSIVE seq(ii) AS (" - " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" - ")" - "SELECT pgno, child FROM sqlite_dbptr('getpage()') " - " UNION ALL " - "SELECT NULL, ii FROM seq", p->laf.nPg - ); - pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, - "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" - ); +/* +** Initialize a recovery handle that returns recovered data in the +** form of SQL statements via a callback. +*/ +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pSqlCtx +){ + return recoverInit(db, zDb, 0, xSql, pSqlCtx); } /* -** Perform one step (sqlite3_recover_step()) of work for the connection -** passed as the only argument, which is guaranteed to be in -** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified -** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged -** to the same database tree. -*/ -static int recoverLostAndFound2Step(sqlite3_recover *p){ - RecoverStateLAF *pLaf = &p->laf; - if( p->errCode==SQLITE_OK ){ - int res = sqlite3_step(pLaf->pAllAndParent); - if( res==SQLITE_ROW ){ - i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); - if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ - sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); - sqlite3_bind_value(pLaf->pMapInsert, 2, - sqlite3_column_value(pLaf->pAllAndParent, 0) - ); - sqlite3_step(pLaf->pMapInsert); - recoverReset(p, pLaf->pMapInsert); - sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); - if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ - int nMax = sqlite3_column_int(pLaf->pMaxField, 0); - if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; - } - recoverReset(p, pLaf->pMaxField); - } - }else{ - recoverFinalize(p, pLaf->pAllAndParent); - pLaf->pAllAndParent =0; - return SQLITE_DONE; - } - } - return p->errCode; +** Return the handle error message, if any. +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; } /* -** Free all resources allocated as part of sqlite3_recover_step() calls -** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. +** Return the handle error code. */ -static void recoverLostAndFoundCleanup(sqlite3_recover *p){ - recoverBitmapFree(p->laf.pUsed); - p->laf.pUsed = 0; - sqlite3_finalize(p->laf.pUsedPages); - sqlite3_finalize(p->laf.pAllAndParent); - sqlite3_finalize(p->laf.pMapInsert); - sqlite3_finalize(p->laf.pMaxField); - sqlite3_finalize(p->laf.pFindRoot); - sqlite3_finalize(p->laf.pInsert); - sqlite3_finalize(p->laf.pAllPage); - sqlite3_finalize(p->laf.pPageData); - p->laf.pUsedPages = 0; - p->laf.pAllAndParent = 0; - p->laf.pMapInsert = 0; - p->laf.pMaxField = 0; - p->laf.pFindRoot = 0; - p->laf.pInsert = 0; - p->laf.pAllPage = 0; - p->laf.pPageData = 0; - sqlite3_free(p->laf.apVal); - p->laf.apVal = 0; +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; } /* -** Free all resources allocated as part of sqlite3_recover_step() calls. +** Configure the handle. */ -static void recoverFinalCleanup(sqlite3_recover *p){ - RecoverTable *pTab = 0; - RecoverTable *pNext = 0; +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else if( p->eState!=RECOVER_STATE_INIT ){ + rc = SQLITE_MISUSE; + }else{ + switch( op ){ + case 789: + /* This undocumented magic configuration option is used to set the + ** name of the auxiliary database that is ATTACH-ed to the database + ** connection and used to hold state information during the + ** recovery process. This option is for debugging use only and + ** is subject to change or removal at any time. */ + sqlite3_free(p->zStateDb); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); + break; - recoverWriteDataCleanup(p); - recoverLostAndFoundCleanup(p); + case SQLITE_RECOVER_LOST_AND_FOUND: { + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); + }else{ + p->zLostAndFound = 0; + } + break; + } - for(pTab=p->pTblList; pTab; pTab=pNext){ - pNext = pTab->pNext; - sqlite3_free(pTab); - } - p->pTblList = 0; - sqlite3_finalize(p->pGetPage); - p->pGetPage = 0; - sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = *(int*)pArg; + break; - { -#ifndef NDEBUG - int res = -#endif - sqlite3_close(p->dbOut); - assert( res==SQLITE_OK ); + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = *(int*)pArg; + break; + + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; + break; + + default: + rc = SQLITE_NOTFOUND; + break; + } } - p->dbOut = 0; + + return rc; } /* -** Decode and return an unsigned 16-bit big-endian integer value from -** buffer a[]. +** Do a unit of work towards the recovery job. Return SQLITE_OK if +** no error has occurred but database recovery is not finished, SQLITE_DONE +** if database recovery has been successfully completed, or an SQLite +** error code if an error has occurred. */ -static u32 recoverGetU16(const u8 *a){ - return (((u32)a[0])<<8) + ((u32)a[1]); +int sqlite3_recover_step(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + if( p->errCode==SQLITE_OK ) recoverStep(p); + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ + return SQLITE_DONE; + } + return p->errCode; } /* -** Decode and return an unsigned 32-bit big-endian integer value from -** buffer a[]. +** Do the configured recovery operation. Return SQLITE_OK if successful, or +** else an SQLite error code. */ -static u32 recoverGetU32(const u8 *a){ - return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +int sqlite3_recover_run(sqlite3_recover *p){ + while( SQLITE_OK==sqlite3_recover_step(p) ); + return sqlite3_recover_errcode(p); } + /* -** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) -** and return the number of bytes consumed. +** Free all resources associated with the recover handle passed as the only +** argument. The results of using a handle with any sqlite3_recover_** +** API function after it has been passed to this function are undefined. +** +** A copy of the value returned by the first call made to sqlite3_recover_run() +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has +** not been called on this handle. */ -static int recoverGetVarint(const u8 *a, i64 *pVal){ - sqlite3_uint64 u = 0; - int i; - for(i=0; i<8; i++){ - u = (u<<7) + (a[i]&0x7f); - if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } +int sqlite3_recover_finish(sqlite3_recover *p){ + int rc; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + recoverFinalCleanup(p); + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ + rc = recoverOneStmt(p->dbIn, "END"); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + } + rc = p->errCode; + sqlite3_free(p->zErrMsg); + sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); + sqlite3_free(p->pPage1Cache); + sqlite3_free(p); } - u = (u<<8) + (a[i]&0xff); - *pVal = (sqlite3_int64)u; - return 9; + return rc; } +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ext/recover/sqlite3recover.c ********************/ +# endif /* SQLITE_HAVE_SQLITE3R */ +#endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif + +#if defined(SQLITE_ENABLE_SESSION) /* -** The second argument points to a buffer n bytes in size. If this buffer -** or a prefix thereof appears to contain a well-formed SQLite b-tree page, -** return the page-size in bytes. Otherwise, if the buffer does not -** appear to contain a well-formed b-tree page, return 0. +** State information for a single open session */ -static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ - u8 *aUsed = aTmp; - int nFrag = 0; - int nActual = 0; - int iFree = 0; - int nCell = 0; /* Number of cells on page */ - int iCellOff = 0; /* Offset of cell array in page */ - int iContent = 0; - int eType = 0; - int ii = 0; - - eType = (int)a[0]; - if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; +typedef struct OpenSession OpenSession; +struct OpenSession { + char *zName; /* Symbolic name for this session */ + int nFilter; /* Number of xFilter rejection GLOB patterns */ + char **azFilter; /* Array of xFilter rejection GLOB patterns */ + sqlite3_session *p; /* The open session */ +}; +#endif - iFree = (int)recoverGetU16(&a[1]); - nCell = (int)recoverGetU16(&a[3]); - iContent = (int)recoverGetU16(&a[5]); - if( iContent==0 ) iContent = 65536; - nFrag = (int)a[7]; +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) +typedef struct ExpertInfo ExpertInfo; +struct ExpertInfo { + sqlite3expert *pExpert; + int bVerbose; +}; +#endif - if( iContent>n ) return 0; +/* All the parameters that determine how to render query results. +*/ +typedef struct Mode { + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ + u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ + u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ + u8 eMode; /* One of the MODE_ values */ + sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ +} Mode; - memset(aUsed, 0, n); - memset(aUsed, 0xFF, iContent); +/* Flags for Mode.mFlags */ +#define MFLG_ECHO 0x01 /* Echo inputs to output */ +#define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ +#define MFLG_HDR 0x04 /* .header used to change headers on/off */ - /* Follow the free-list. This is the same format for all b-tree pages. */ - if( iFree && iFree<=iContent ) return 0; - while( iFree ){ - int iNext = 0; - int nByte = 0; - if( iFree>(n-4) ) return 0; - iNext = recoverGetU16(&a[iFree]); - nByte = recoverGetU16(&a[iFree+2]); - if( iFree+nByte>n || nByte<4 ) return 0; - if( iNext && iNextiContent ) return 0; - for(ii=0; iin ){ - return 0; - } - if( eType==0x05 || eType==0x02 ) nByte += 4; - nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); - if( eType==0x0D ){ - i64 dummy = 0; - nByte += recoverGetVarint(&a[iOff+nByte], &dummy); - } - if( eType!=0x05 ){ - int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); - int M = ((n-12)*32/255)-23; - int K = M+((nPayload-M)%(n-4)); +/* +** State information about the database connection is contained in an +** instance of the following structure. +*/ +typedef struct ShellState ShellState; +struct ShellState { + sqlite3 *db; /* The database */ + u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ + u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ + u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ + u8 bSafeMode; /* True to prohibit unsafe operations */ + u8 bSafeModePersist; /* The long-term value of bSafeMode */ + u8 eRestoreState; /* See comments above doAutoDetectRestore() */ + unsigned statsOn; /* True to display memory stats before each finalize */ + unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ + u8 nPopOutput; /* Revert .output settings when reaching zero */ + u8 nPopMode; /* Revert .mode settings when reaching zero */ + u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ + int inputNesting; /* Track nesting level of .read and other redirects */ + double prevTimer; /* Last reported timer value */ + double tmProgress; /* --timeout option for .progress */ + i64 lineno; /* Line number of last line read from in */ + const char *zInFile; /* Name of the input file */ + int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ + FILE *in; /* Read commands from this stream */ + FILE *out; /* Write results here */ + FILE *traceOut; /* Output for sqlite3_trace() */ + int nErr; /* Number of errors seen */ + int writableSchema; /* True if PRAGMA writable_schema=ON */ + int nCheck; /* Number of ".check" commands run */ + unsigned nProgress; /* Number of progress callbacks encountered */ + unsigned mxProgress; /* Maximum progress callbacks before failing */ + unsigned flgProgress; /* Flags for the progress callback */ + unsigned shellFlgs; /* Various flags */ + unsigned nTestRun; /* Number of test cases run */ + unsigned nTestErr; /* Number of test cases that failed */ + sqlite3_int64 szMax; /* --maxsize argument to .open */ + char *zDestTable; /* Name of destination table when MODE_Insert */ + char *zTempFile; /* Temporary file that might need deleting */ + char *zErrPrefix; /* Alternative error message prefix */ + char zTestcase[30]; /* Name of current test case */ + char outfile[FILENAME_MAX]; /* Filename for *out */ + sqlite3_stmt *pStmt; /* Current statement if any. */ + FILE *pLog; /* Write log output here */ + Mode mode; /* Current display mode */ + Mode modePrior; /* Backup */ + struct SavedMode { /* Ability to define custom mode configurations */ + char *zTag; /* Name of this saved mode */ + Mode mode; /* The saved mode */ + } *aSavedModes; /* Array of saved .mode settings. system malloc() */ + int nSavedModes; /* Number of saved .mode settings */ + struct AuxDb { /* Storage space for auxiliary database connections */ + sqlite3 *db; /* Connection pointer */ + const char *zDbFilename; /* Filename used to open the connection */ + char *zFreeOnClose; /* Free this memory allocation on close */ +#if defined(SQLITE_ENABLE_SESSION) + int nSession; /* Number of active sessions */ + OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +#endif + } aAuxDb[5], /* Array of all database connections */ + *pAuxDb; /* Currently active database connection */ + char *zNonce; /* Nonce for temporary safe-mode escapes */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) + ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +#endif + struct DotCmdLine { /* Info about arguments to a dot-command */ + const char *zOrig; /* Original text of the dot-command */ + char *zCopy; /* Copy of zOrig, from malloc() */ + int nAlloc; /* Size of allocates for arrays below */ + int nArg; /* Number of argument slots actually used */ + char **azArg; /* Pointer to each argument, dequoted */ + int *aiOfst; /* Offset into zOrig[] for start of each arg */ + char *abQuot; /* True if the argment was originally quoted */ + } dot; +#ifdef SQLITE_SHELL_FIDDLE + struct { + const char * zInput; /* Input string from wasm/JS proxy */ + const char * zPos; /* Cursor pos into zInput */ + const char * zDefaultDbName; /* Default name for db file */ + } wasm; +#endif +}; - if( nPayloadn ){ - return 0; - } - for(iByte=iOff; iByte<(iOff+nByte); iByte++){ - if( aUsed[iByte]!=0 ){ - return 0; - } - aUsed[iByte] = 0xFF; - } - } - nActual = 0; - for(ii=0; iipMethods!=&recover_methods ); - return pFd->pMethods->xClose(pFd); -} +/* Names of values for Mode.spec.eEsc and Mode.spec.eText +*/ +static const char *qrfEscNames[] = { "auto", "off", "ascii", "symbol" }; +static const char *qrfQuoteNames[] = + { "off","off","sql","hex","csv","tcl","json","relaxed"}; /* -** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. +** These are the allowed shellFlgs values */ -static void recoverPutU16(u8 *a, u32 v){ - a[0] = (v>>8) & 0x00FF; - a[1] = (v>>0) & 0x00FF; -} +#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ +#define SHFLG_NoErrLineno 0x00000010 /* Omit line numbers from error msgs */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ +#define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ +#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ /* -** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +** Macros for testing and setting shellFlgs */ -static void recoverPutU32(u8 *a, u32 v){ - a[0] = (v>>24) & 0x00FF; - a[1] = (v>>16) & 0x00FF; - a[2] = (v>>8) & 0x00FF; - a[3] = (v>>0) & 0x00FF; -} +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* -** Detect the page-size of the database opened by file-handle pFd by -** searching the first part of the file for a well-formed SQLite b-tree -** page. If parameter nReserve is non-zero, then as well as searching for -** a b-tree page with zero reserved bytes, this function searches for one -** with nReserve reserved bytes at the end of it. -** -** If successful, set variable p->detected_pgsz to the detected page-size -** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page -** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, -** if an error occurs (e.g. an IO or OOM error), then an SQLite error code -** is returned. The final value of p->detected_pgsz is undefined in this -** case. +** These are the allowed values for Mode.eMode. There is a lot of overlap +** between these values and the Mode.spec.eStyle values, but they are not +** one-to-one, and thus need to be tracked separately. */ -static int recoverVfsDetectPagesize( - sqlite3_recover *p, /* Recover handle */ - sqlite3_file *pFd, /* File-handle open on input database */ - u32 nReserve, /* Possible nReserve value */ - i64 nSz /* Size of database file in bytes */ -){ - int rc = SQLITE_OK; - const int nMin = 512; - const int nMax = 65536; - const int nMaxBlk = 4; - u32 pgsz = 0; - int iBlk = 0; - u8 *aPg = 0; - u8 *aTmp = 0; - int nBlk = 0; +#define MODE_Ascii 0 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Box 1 /* Unicode box-drawing characters */ +#define MODE_C 2 /* Comma-separated list of C-strings */ +#define MODE_Column 3 /* One record per line in neat columns */ +#define MODE_Count 4 /* Output only a count of the rows of output */ +#define MODE_Csv 5 /* Quote strings, numbers are plain */ +#define MODE_Html 6 /* Generate an XHTML table */ +#define MODE_Insert 7 /* Generate SQL "insert" statements */ +#define MODE_JAtom 8 /* Comma-separated list of JSON atoms */ +#define MODE_JObject 9 /* One JSON object per row */ +#define MODE_Json 10 /* Output JSON */ +#define MODE_Line 11 /* One column per line. Blank line between records */ +#define MODE_List 12 /* One record per line with a separator */ +#define MODE_Markdown 13 /* Markdown formatting */ +#define MODE_Off 14 /* No query output shown */ +#define MODE_Psql 15 /* Similar to psql */ +#define MODE_QBox 16 /* BOX with SQL-quoted content */ +#define MODE_Quote 17 /* Quote values as for SQL */ +#define MODE_Split 18 /* Split-column mode */ +#define MODE_Table 19 /* MySQL-style table formatting */ +#define MODE_Tabs 20 /* Tab-separated values */ +#define MODE_Tcl 21 /* Space-separated list of TCL strings */ +#define MODE_Www 22 /* Full web-page output */ - aPg = (u8*)sqlite3_malloc(2*nMax); - if( aPg==0 ) return SQLITE_NOMEM; - aTmp = &aPg[nMax]; +#define MODE_BUILTIN 22 /* Maximum built-in mode */ +#define MODE_BATCH 50 /* Default mode for batch processing */ +#define MODE_TTY 51 /* Default mode for interactive processing */ +#define MODE_USER 75 /* First user-defined mode */ +#define MODE_N_USER 25 /* Maximum number of user-defined modes */ - nBlk = (nSz+nMax-1)/nMax; - if( nBlk>nMaxBlk ) nBlk = nMaxBlk; +/* +** Information about built-in display modes +*/ +typedef struct ModeInfo ModeInfo; +struct ModeInfo { + char zName[9]; /* Symbolic name of the mode */ + unsigned char eCSep; /* Column separator */ + unsigned char eRSep; /* Row separator */ + unsigned char eNull; /* Null representation */ + unsigned char eText; /* Default text encoding */ + unsigned char eHdr; /* Default header encoding. */ + unsigned char eBlob; /* Default blob encoding. */ + unsigned char bHdr; /* Show headers by default. 0: n/a, 1: no 2: yes */ + unsigned char eStyle; /* Underlying QRF style */ + unsigned char eCx; /* 0: other, 1: line, 2: columnar */ + unsigned char mFlg; /* Flags. 1=border-off 2=split-column */ +}; - do { - for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); - memset(aPg, 0, nMax); - rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); - if( rc==SQLITE_OK ){ - int pgsz2; - for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ - int iOff; - for(iOff=0; iOff(u32)p->detected_pgsz ){ - p->detected_pgsz = pgsz; - p->nReserve = nReserve; - } - if( nReserve==0 ) break; - nReserve = 0; - }while( 1 ); +/* String constants used by built-in modes */ +static const char *aModeStr[] = + /* 0 1 2 3 4 5 6 7 8 */ + { 0, "\n", "|", " ", ",", "\r\n", "\036", "\037", "\t", + "", "NULL", "null", "\"\"", ": ", }; + /* 9 10 11 12 13 */ - p->detected_pgsz = pgsz; - sqlite3_free(aPg); - return rc; -} +static const ModeInfo aModeInfo[] = { +/* zName eCSep eRSep eNull eText eHdr eBlob bHdr eStyle eCx mFlg */ + { "ascii", 7, 6, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "box", 0, 0, 9, 1, 1, 0, 2, 1, 2, 0 }, + { "c", 4, 1, 10, 5, 5, 4, 1, 12, 0, 0 }, + { "column", 0, 0, 9, 1, 1, 0, 2, 2, 2, 0 }, + { "count", 0, 0, 0, 0, 0, 0, 0, 3, 0, 0 }, + { "csv", 4, 5, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "html", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 }, + { "insert", 0, 0, 10, 2, 2, 0, 1, 8, 0, 0 }, + { "jatom", 4, 1, 11, 6, 6, 0, 1, 12, 0, 0 }, + { "jobject", 0, 1, 11, 6, 6, 0, 0, 10, 0, 0 }, + { "json", 0, 0, 11, 6, 6, 0, 0, 9, 0, 0 }, + { "line", 13, 1, 9, 1, 1, 0, 0, 11, 1, 0 }, + { "list", 2, 1, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "markdown", 0, 0, 9, 1, 1, 0, 2, 13, 2, 0 }, + { "off", 0, 0, 0, 0, 0, 0, 0, 14, 0, 0 }, + { "psql", 0, 0, 9, 1, 1, 0, 2, 19, 2, 1 }, + { "qbox", 0, 0, 10, 2, 1, 0, 2, 1, 2, 0 }, + { "quote", 4, 1, 10, 2, 2, 0, 1, 12, 0, 0 }, + { "split", 0, 0, 9, 1, 1, 0, 1, 2, 2, 2 }, + { "table", 0, 0, 9, 1, 1, 0, 2, 19, 2, 0 }, + { "tabs", 8, 1, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "tcl", 3, 1, 12, 5, 5, 4, 1, 12, 0, 0 }, + { "www", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 } +}; /* | / / | / / | | \ + ** | / / | / / | | \_ 2: columnar + ** Index into aModeStr[] | / / | | 1: line + ** | / / | | 0: other + ** | / / | \ + ** text encoding |/ | show | \ + ** v-------------------' | hdrs? | The QRF style + ** 0: n/a blob | v-----' + ** 1: plain v_---------' 0: n/a + ** 2: sql 0: auto 1: no + ** 3: csv 1: as-text 2: yes + ** 4: html 2: sql + ** 5: c 3: hex + ** 6: json 4: c + ** 5: json + ** 6: size + ******************************************************************/ +/* +** These are the column/row/line separators used by the various +** import/export modes. +*/ +#define SEP_Column "|" +#define SEP_Row "\n" +#define SEP_Tab "\t" +#define SEP_Space " " +#define SEP_Comma "," +#define SEP_CrLf "\r\n" +#define SEP_Unit "\x1F" +#define SEP_Record "\x1E" + +/* +** Default values for the various QRF limits +*/ +#ifndef DFLT_CHAR_LIMIT +# define DFLT_CHAR_LIMIT 300 +#endif +#ifndef DFLT_LINE_LIMIT +# define DFLT_LINE_LIMIT 5 +#endif +#ifndef DFLT_TITLE_LIMIT +# define DFLT_TITLE_LIMIT 20 +#endif +#ifndef DFLT_MULTI_INSERT +# define DFLT_MULTI_INSERT 3000 +#endif /* -** The xRead() method of the wrapper VFS. This is used to intercept calls -** to read page 1 of the input database. +** If the following flag is set, then command execution stops +** at an error if we are not interactive. */ -static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ - int rc = SQLITE_OK; - if( pFd->pMethods==&recover_methods ){ - pFd->pMethods = recover_g.pMethods; - rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); - if( nByte==16 ){ - sqlite3_randomness(16, aBuf); - }else - if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ - /* Ensure that the database has a valid header file. The only fields - ** that really matter to recovery are: - ** - ** + Database page size (16-bits at offset 16) - ** + Size of db in pages (32-bits at offset 28) - ** + Database encoding (32-bits at offset 56) - ** - ** Also preserved are: - ** - ** + first freelist page (32-bits at offset 32) - ** + size of freelist (32-bits at offset 36) - ** + the wal-mode flags (16-bits at offset 18) - ** - ** We also try to preserve the auto-vacuum, incr-value, user-version - ** and application-id fields - all 32 bit quantities at offsets - ** 52, 60, 64 and 68. All other fields are set to known good values. - ** - ** Byte offset 105 should also contain the page-size as a 16-bit - ** integer. - */ - const int aPreserve[] = {32, 36, 52, 60, 64, 68}; - u8 aHdr[108] = { - 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, - 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, - 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x2e, 0x5b, 0x30, - - 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 - }; - u8 *a = (u8*)aBuf; - - u32 pgsz = recoverGetU16(&a[16]); - u32 nReserve = a[20]; - u32 enc = recoverGetU32(&a[56]); - u32 dbsz = 0; - i64 dbFileSize = 0; - int ii; - sqlite3_recover *p = recover_g.p; +static int bail_on_error = 0; - if( pgsz==0x01 ) pgsz = 65536; - rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); +/* +** Treat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. +*/ +static int stdin_is_interactive = 1; - if( rc==SQLITE_OK && p->detected_pgsz==0 ){ - rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); - } - if( p->detected_pgsz ){ - pgsz = p->detected_pgsz; - nReserve = p->nReserve; - } +/* +** Treat stdout like a TTY if true. +*/ +static int stdout_is_console = 1; - if( pgsz ){ - dbsz = dbFileSize / pgsz; - } - if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ - enc = SQLITE_UTF8; - } +/* +** Use this value as the width of the output device. Or, figure it +** out at runtime if the value is negative. Or use a default width +** if this value is zero. +*/ +static int stdout_tty_width = -1; - sqlite3_free(p->pPage1Cache); - p->pPage1Cache = 0; - p->pPage1Disk = 0; +/* +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. +*/ +static sqlite3 *globalDb = 0; - p->pgsz = nByte; - p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); - if( p->pPage1Cache ){ - p->pPage1Disk = &p->pPage1Cache[nByte]; - memcpy(p->pPage1Disk, aBuf, nByte); - aHdr[18] = a[18]; - aHdr[19] = a[19]; - recoverPutU32(&aHdr[28], dbsz); - recoverPutU32(&aHdr[56], enc); - recoverPutU16(&aHdr[105], pgsz-nReserve); - if( pgsz==65536 ) pgsz = 1; - recoverPutU16(&aHdr[16], pgsz); - aHdr[20] = nReserve; - for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){ - memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4); - } - memcpy(aBuf, aHdr, sizeof(aHdr)); - memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr)); +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; - memcpy(p->pPage1Cache, aBuf, nByte); - }else{ - rc = p->errCode; - } +/* +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. +*/ +static char *Argv0; - } - pFd->pMethods = &recover_methods; - }else{ - rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); - } - return rc; -} +/* +** Prompt strings. Initialized in main. Settable with +** .prompt main continue +*/ +#define PROMPT_LEN_MAX 128 +/* First line prompt. default: "sqlite> " */ +static char mainPrompt[PROMPT_LEN_MAX]; +/* Continuation prompt. default: " ...> " */ +static char continuePrompt[PROMPT_LEN_MAX]; /* -** Used to make sqlite3_io_methods wrapper methods less verbose. +** Write I/O traces to the following stream. */ -#define RECOVER_VFS_WRAPPER(code) \ - int rc = SQLITE_OK; \ - if( pFd->pMethods==&recover_methods ){ \ - pFd->pMethods = recover_g.pMethods; \ - rc = code; \ - pFd->pMethods = &recover_methods; \ - }else{ \ - rc = code; \ - } \ - return rc; +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif /* -** Methods of the wrapper VFS. All methods except for xRead() and xClose() -** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent -** method on the lower level VFS, then reinstall the wrapper before returning. -** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +** Output routines that are able to redirect to memory rather than +** doing actually I/O. +** Works like. +** -------------- +** cli_printf(FILE*, const char*, ...); fprintf() +** cli_puts(const char*, FILE*); fputs() +** cli_vprintf(FILE*, const char*, va_list); vfprintf() +** +** These are just thin wrappers with the following added semantics: +** If the file-scope variable cli_output_capture is not NULL, and +** if the FILE* argument is stdout or stderr, then rather than +** writing to stdout/stdout, append the text to the cli_output_capture +** variable. +** +** The cli_exit(int) routine works like exit() except that it +** first dumps any capture output to stdout. */ -static int recoverVfsWrite( - sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff -){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) - ); -} -static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xTruncate(pFd, size) - ); -} -static int recoverVfsSync(sqlite3_file *pFd, int flags){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xSync(pFd, flags) - ); -} -static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xFileSize(pFd, pSize) - ); -} -static int recoverVfsLock(sqlite3_file *pFd, int eLock){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xLock(pFd, eLock) - ); -} -static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xUnlock(pFd, eLock) - ); -} -static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xCheckReservedLock(pFd, pResOut) - ); -} -static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ - RECOVER_VFS_WRAPPER ( - (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) - ); -} -static int recoverVfsSectorSize(sqlite3_file *pFd){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xSectorSize(pFd) - ); -} -static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xDeviceCharacteristics(pFd) - ); -} -static int recoverVfsShmMap( - sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp -){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) - ); +static sqlite3_str *cli_output_capture = 0; +static int cli_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + int rc; + va_start(ap,zFormat); + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + rc = 1; + }else{ + rc = sqlite3_vfprintf(out, zFormat, ap); + } + va_end(ap); + return rc; } -static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xShmLock(pFd, offset, n, flags) - ); +static int cli_puts(const char *zText, FILE *out){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_appendall(cli_output_capture, zText); + return 1; + } + return sqlite3_fputs(zText, out); } -static void recoverVfsShmBarrier(sqlite3_file *pFd){ - if( pFd->pMethods==&recover_methods ){ - pFd->pMethods = recover_g.pMethods; - pFd->pMethods->xShmBarrier(pFd); - pFd->pMethods = &recover_methods; +#if 0 /* Not currently used - available if we need it later */ +static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + return 1; }else{ - pFd->pMethods->xShmBarrier(pFd); + return sqlite3_vfprintf(out, zFormat, ap); } } -static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ - RECOVER_VFS_WRAPPER ( - pFd->pMethods->xShmUnmap(pFd, deleteFlag) - ); +#endif +static void cli_exit(int rc){ + if( cli_output_capture ){ + char *z = sqlite3_str_finish(cli_output_capture); + sqlite3_fputs(z, stdout); + fflush(stdout); + } + exit(rc); } -static int recoverVfsFetch( - sqlite3_file *pFd, - sqlite3_int64 iOff, - int iAmt, - void **pp -){ - (void)pFd; - (void)iOff; - (void)iAmt; - *pp = 0; - return SQLITE_OK; + +#define eputz(z) cli_puts(z,stderr) +#define sputz(fp,z) cli_puts(z,fp) + +/* A version of strcmp() that works with NULL values */ +static int cli_strcmp(const char *a, const char *b){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strcmp(a,b); } -static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ - (void)pFd; - (void)iOff; - (void)p; - return SQLITE_OK; +static int cli_strncmp(const char *a, const char *b, size_t n){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strncmp(a,b,n); } -/* -** Install the VFS wrapper around the file-descriptor open on the input -** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held -** when this function is called. +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) */ -static void recoverInstallWrapper(sqlite3_recover *p){ - sqlite3_file *pFd = 0; - assert( recover_g.pMethods==0 ); - recoverAssertMutexHeld(); - sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); - assert( pFd==0 || pFd->pMethods!=&recover_methods ); - if( pFd && pFd->pMethods ){ - int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); - recover_g.pMethods = pFd->pMethods; - recover_g.p = p; - recover_methods.iVersion = iVersion; - pFd->pMethods = &recover_methods; +static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; /* Never actually happens */ + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); } + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif } -/* -** Uninstall the VFS wrapper that was installed around the file-descriptor open -** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be -** held when this function is called. + + +/* This is variant of the standard-library strncpy() routine with the +** one change that the destination string is always zero-terminated, even +** if there is no zero-terminator in the first n-1 characters of the source +** string. */ -static void recoverUninstallWrapper(sqlite3_recover *p){ - sqlite3_file *pFd = 0; - recoverAssertMutexHeld(); - sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); - if( pFd && pFd->pMethods ){ - pFd->pMethods = recover_g.pMethods; - recover_g.pMethods = 0; - recover_g.p = 0; - } +static char *shell_strncpy(char *dest, const char *src, size_t n){ + size_t i; + for(i=0; ierrCode==SQLITE_OK ); - switch( p->eState ){ - case RECOVER_STATE_INIT: { - int bUseWrapper = 1; - /* This is the very first call to sqlite3_recover_step() on this object. - */ - recoverSqlCallback(p, "BEGIN"); - recoverSqlCallback(p, "PRAGMA writable_schema = on"); - recoverSqlCallback(p, "PRAGMA foreign_keys = off"); - - recoverEnterMutex(); - - /* Open the output database. And register required virtual tables and - ** user functions with the new handle. */ - recoverOpenOutput(p); - - /* Attempt to open a transaction and read page 1 of the input database. - ** Two attempts may be made - one with a wrapper installed to ensure - ** that the database header is sane, and then if that attempt returns - ** SQLITE_NOTADB, then again with no wrapper. The second attempt is - ** required for encrypted databases. */ - if( p->errCode==SQLITE_OK ){ - do{ - p->errCode = SQLITE_OK; - if( bUseWrapper ) recoverInstallWrapper(p); - - /* Open a transaction on the input database. */ - sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); - recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); - recoverExec(p, p->dbIn, "BEGIN"); - if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; - recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); - recoverTransferSettings(p); - recoverOpenRecovery(p); - recoverCacheSchema(p); - - if( bUseWrapper ) recoverUninstallWrapper(p); - }while( p->errCode==SQLITE_NOTADB - && (bUseWrapper--) - && SQLITE_OK==sqlite3_exec(p->dbIn, "ROLLBACK", 0, 0, 0) - ); - } - - recoverLeaveMutex(); - recoverExec(p, p->dbOut, "BEGIN"); - recoverWriteSchema1(p); - p->eState = RECOVER_STATE_WRITING; - break; - } - - case RECOVER_STATE_WRITING: { - if( p->w1.pTbls==0 ){ - recoverWriteDataInit(p); - } - if( SQLITE_DONE==recoverWriteDataStep(p) ){ - recoverWriteDataCleanup(p); - if( p->zLostAndFound ){ - p->eState = RECOVER_STATE_LOSTANDFOUND1; - }else{ - p->eState = RECOVER_STATE_SCHEMA2; - } - } - break; - } - - case RECOVER_STATE_LOSTANDFOUND1: { - if( p->laf.pUsed==0 ){ - recoverLostAndFound1Init(p); - } - if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ - p->eState = RECOVER_STATE_LOSTANDFOUND2; - } - break; - } - case RECOVER_STATE_LOSTANDFOUND2: { - if( p->laf.pAllAndParent==0 ){ - recoverLostAndFound2Init(p); - } - if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ - p->eState = RECOVER_STATE_LOSTANDFOUND3; - } - break; - } - - case RECOVER_STATE_LOSTANDFOUND3: { - if( p->laf.pInsert==0 ){ - recoverLostAndFound3Init(p); - } - if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ - p->eState = RECOVER_STATE_SCHEMA2; - } - break; - } - - case RECOVER_STATE_SCHEMA2: { - int rc = SQLITE_OK; - - recoverWriteSchema2(p); - p->eState = RECOVER_STATE_DONE; - - /* If no error has occurred, commit the write transaction on the output - ** database. Regardless of whether or not an error has occurred, make - ** an attempt to end the read transaction on the input database. */ - recoverExec(p, p->dbOut, "COMMIT"); - rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); - if( p->errCode==SQLITE_OK ) p->errCode = rc; - - recoverSqlCallback(p, "PRAGMA writable_schema = off"); - recoverSqlCallback(p, "COMMIT"); - p->eState = RECOVER_STATE_DONE; - recoverFinalCleanup(p); - break; - }; - - case RECOVER_STATE_DONE: { - /* no-op */ - break; - }; - } +** strcpy() workalike to squelch an unwarranted link-time warning +** from OpenBSD. +*/ +static void shell_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} } - /* -** This is a worker function that does the heavy lifting for both init -** functions: -** -** sqlite3_recover_init() -** sqlite3_recover_init_sql() -** -** All this function does is allocate space for the recover handle and -** take copies of the input parameters. All the real work is done within -** sqlite3_recover_run(). +** Optionally disable dynamic continuation prompt. +** Unless disabled, the continuation prompt shows open SQL lexemes if any, +** or open parentheses level if non-zero, or continuation prompt as set. +** This facility interacts with the scanner and process_input() where the +** below 5 macros are used. */ -sqlite3_recover *recoverInit( - sqlite3* db, - const char *zDb, - const char *zUri, /* Output URI for _recover_init() */ - int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ - void *pSqlCtx /* Context arg for _recover_init_sql() */ -){ - sqlite3_recover *pRet = 0; - int nDb = 0; - int nUri = 0; - int nByte = 0; +#ifdef SQLITE_OMIT_DYNAPROMPT +# define CONTINUATION_PROMPT continuePrompt +# define CONTINUE_PROMPT_RESET +# define CONTINUE_PROMPT_AWAITS(p,s) +# define CONTINUE_PROMPT_AWAITC(p,c) +# define CONTINUE_PAREN_INCR(p,n) +# define CONTINUE_PROMPT_PSTATE 0 +typedef void *t_NoDynaPrompt; +# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt +#else +# define CONTINUATION_PROMPT dynamicContinuePrompt() +# define CONTINUE_PROMPT_RESET \ + do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) +# define CONTINUE_PROMPT_AWAITS(p,s) \ + if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) +# define CONTINUE_PROMPT_AWAITC(p,c) \ + if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) +# define CONTINUE_PAREN_INCR(p,n) \ + if(p && stdin_is_interactive) (trackParenLevel(p,n)) +# define CONTINUE_PROMPT_PSTATE (&dynPrompt) +typedef struct DynaPrompt *t_DynaPromptRef; +# define SCAN_TRACKER_REFTYPE t_DynaPromptRef - if( zDb==0 ){ zDb = "main"; } +static struct DynaPrompt { + char dynamicPrompt[PROMPT_LEN_MAX]; + char acAwait[2]; + int inParenLevel; + char *zScannerAwaits; +} dynPrompt = { {0}, {0}, 0, 0 }; - nDb = recoverStrlen(zDb); - nUri = recoverStrlen(zUri); +/* Record parenthesis nesting level change, or force level to 0. */ +static void trackParenLevel(struct DynaPrompt *p, int ni){ + p->inParenLevel += ni; + if( ni==0 ) p->inParenLevel = 0; + p->zScannerAwaits = 0; +} - nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; - pRet = (sqlite3_recover*)sqlite3_malloc(nByte); - if( pRet ){ - memset(pRet, 0, nByte); - pRet->dbIn = db; - pRet->zDb = (char*)&pRet[1]; - pRet->zUri = &pRet->zDb[nDb+1]; - memcpy(pRet->zDb, zDb, nDb); - if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); - pRet->xSql = xSql; - pRet->pSqlCtx = pSqlCtx; - pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; +/* Record that a lexeme is opened, or closed with args==0. */ +static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ + if( s!=0 || c==0 ){ + p->zScannerAwaits = s; + p->acAwait[0] = 0; + }else{ + p->acAwait[0] = c; + p->zScannerAwaits = p->acAwait; } +} - return pRet; +/* Upon demand, derive the continuation prompt to display. */ +static char *dynamicContinuePrompt(void){ + if( continuePrompt[0]==0 + || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ + return continuePrompt; + }else{ + if( dynPrompt.zScannerAwaits ){ + size_t ncp = strlen(continuePrompt); + size_t ndp = strlen(dynPrompt.zScannerAwaits); + if( ndp > ncp-3 ) return continuePrompt; + shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + }else{ + if( dynPrompt.inParenLevel>9 ){ + shell_strncpy(dynPrompt.dynamicPrompt, "(..", 4); + }else if( dynPrompt.inParenLevel<0 ){ + shell_strncpy(dynPrompt.dynamicPrompt, ")x!", 4); + }else{ + shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); + dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + } + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + } + } + return dynPrompt.dynamicPrompt; } +#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ -/* -** Initialize a recovery handle that creates a new database containing -** the recovered data. -*/ -sqlite3_recover *sqlite3_recover_init( - sqlite3* db, - const char *zDb, - const char *zUri -){ - return recoverInit(db, zDb, zUri, 0, 0); +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + eputz("Error: out of memory\n"); + cli_exit(1); } -/* -** Initialize a recovery handle that returns recovered data in the -** form of SQL statements via a callback. +/* Check a pointer to see if it is NULL. If it is NULL, exit with an +** out-of-memory error. */ -sqlite3_recover *sqlite3_recover_init_sql( - sqlite3* db, - const char *zDb, - int (*xSql)(void*, const char*), - void *pSqlCtx -){ - return recoverInit(db, zDb, 0, xSql, pSqlCtx); +static void shell_check_oom(const void *p){ + if( p==0 ) shell_out_of_memory(); } /* -** Return the handle error message, if any. +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. */ -const char *sqlite3_recover_errmsg(sqlite3_recover *p){ - return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; +#ifdef SQLITE_ENABLE_IOTRACE +static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + cli_printf(iotrace, "%s", z); + sqlite3_free(z); } +#endif /* -** Return the handle error code. +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. */ -int sqlite3_recover_errcode(sqlite3_recover *p){ - return p ? p->errCode : SQLITE_NOMEM; +static int strlen30(const char *z){ + size_t n; + if( z==0 ) return 0; + n = strlen(z); + return n>0x3fffffff ? 0x3fffffff : (int)n; } /* -** Configure the handle. +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. */ -int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ - int rc = SQLITE_OK; - if( p==0 ){ - rc = SQLITE_NOMEM; - }else if( p->eState!=RECOVER_STATE_INIT ){ - rc = SQLITE_MISUSE; +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = sqlite3_fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; + } + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return sqlite3_fopen(zFile, "rb"); }else{ - switch( op ){ - case 789: - /* This undocumented magic configuration option is used to set the - ** name of the auxiliary database that is ATTACH-ed to the database - ** connection and used to hold state information during the - ** recovery process. This option is for debugging use only and - ** is subject to change or removal at any time. */ - sqlite3_free(p->zStateDb); - p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); - break; - - case SQLITE_RECOVER_LOST_AND_FOUND: { - const char *zArg = (const char*)pArg; - sqlite3_free(p->zLostAndFound); - if( zArg ){ - p->zLostAndFound = recoverMPrintf(p, "%s", zArg); - }else{ - p->zLostAndFound = 0; - } - break; - } - - case SQLITE_RECOVER_FREELIST_CORRUPT: - p->bFreelistCorrupt = *(int*)pArg; - break; - - case SQLITE_RECOVER_ROWIDS: - p->bRecoverRowid = *(int*)pArg; - break; - - case SQLITE_RECOVER_SLOWINDEXES: - p->bSlowIndexes = *(int*)pArg; - break; - - default: - rc = SQLITE_NOTFOUND; - break; - } + return 0; } - - return rc; +#endif +#undef STAT_CHR_SRC } /* -** Do a unit of work towards the recovery job. Return SQLITE_OK if -** no error has occurred but database recovery is not finished, SQLITE_DONE -** if database recovery has been successfully completed, or an SQLite -** error code if an error has occurred. +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails, or if the length of the line is longer than about a gigabyte. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. */ -int sqlite3_recover_step(sqlite3_recover *p){ - if( p==0 ) return SQLITE_NOMEM; - if( p->errCode==SQLITE_OK ) recoverStep(p); - if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ - return SQLITE_DONE; - } - return p->errCode; -} +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; -/* -** Do the configured recovery operation. Return SQLITE_OK if successful, or -** else an SQLite error code. -*/ -int sqlite3_recover_run(sqlite3_recover *p){ - while( SQLITE_OK==sqlite3_recover_step(p) ); - return sqlite3_recover_errcode(p); + while( 1 ){ + if( n+100>nLine ){ + if( nLine>=1073741773 ){ + free(zLine); + return 0; + } + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + shell_check_oom(zLine); + } + if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; + zLine[n] = 0; + break; + } + } + return zLine; } - /* -** Free all resources associated with the recover handle passed as the only -** argument. The results of using a handle with any sqlite3_recover_** -** API function after it has been passed to this function are undefined. +** Retrieve a single line of input text. ** -** A copy of the value returned by the first call made to sqlite3_recover_run() -** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has -** not been called on this handle. +** If in==0 then read from standard input and prompt before each line. +** If isContinuation is true, then a continuation prompt is appropriate. +** If isContinuation is zero, then the main prompt should be used. +** +** If zPrior is not NULL then it is a buffer from a prior call to this +** routine that can be reused. +** +** The result is stored in space obtained from malloc() and must either +** be freed by the caller or else passed back into this routine via the +** zPrior argument for reuse. */ -int sqlite3_recover_finish(sqlite3_recover *p){ - int rc; - if( p==0 ){ - rc = SQLITE_NOMEM; +#ifndef SQLITE_SHELL_FIDDLE +static char *one_input_line(ShellState *p, char *zPrior, int isContinuation){ + char *zPrompt; + char *zResult; + FILE *in = p->in; + if( in!=0 ){ + zResult = local_getline(zPrior, in); }else{ - recoverFinalCleanup(p); - if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ - rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); - if( p->errCode==SQLITE_OK ) p->errCode = rc; + zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; +#if SHELL_USE_LOCAL_GETLINE + sputz(stdout, zPrompt); + fflush(stdout); + do{ + zResult = local_getline(zPrior, stdin); + zPrior = 0; + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + if( zResult==0 ) sqlite3_sleep(50); + }while( zResult==0 && seenInterrupt>0 ); +#else + free(zPrior); + zResult = shell_readline(zPrompt); + while( zResult==0 ){ + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + sqlite3_sleep(50); + if( seenInterrupt==0 ) break; + zResult = shell_readline(""); } - rc = p->errCode; - sqlite3_free(p->zErrMsg); - sqlite3_free(p->zStateDb); - sqlite3_free(p->zLostAndFound); - sqlite3_free(p->pPage1Cache); - sqlite3_free(p); + if( zResult && *zResult ) shell_add_history(zResult); +#endif } - return rc; + return zResult; } +#endif /* !SQLITE_SHELL_FIDDLE */ -#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ - -/************************* End ext/recover/sqlite3recover.c ********************/ -# endif /* SQLITE_HAVE_SQLITE3R */ -#endif -#ifdef SQLITE_SHELL_EXTSRC -# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) -#endif - -#if defined(SQLITE_ENABLE_SESSION) /* -** State information for a single open session -*/ -typedef struct OpenSession OpenSession; -struct OpenSession { - char *zName; /* Symbolic name for this session */ - int nFilter; /* Number of xFilter rejection GLOB patterns */ - char **azFilter; /* Array of xFilter rejection GLOB patterns */ - sqlite3_session *p; /* The open session */ -}; -#endif - -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) -typedef struct ExpertInfo ExpertInfo; -struct ExpertInfo { - sqlite3expert *pExpert; - int bVerbose; -}; -#endif - -/* All the parameters that determine how to render query results. +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. */ -typedef struct Mode { - u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ - u8 autoEQPtrace; /* autoEQP is in trace mode */ - u8 scanstatsOn; /* True to display scan stats before each finalize */ - u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ - u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ - u8 eMode; /* One of the MODE_ values */ - sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ -} Mode; - -/* Flags for Mode.mFlags */ -#define MFLG_ECHO 0x01 /* Echo inputs to output */ -#define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ -#define MFLG_HDR 0x04 /* .header used to change headers on/off */ - +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} /* -** State information about the database connection is contained in an -** instance of the following structure. +** Interpret zArg as an integer value, possibly with suffixes. +** +** If the value specified by zArg is outside the range of values that +** can be represented using a 64-bit twos-complement integer, then return +** the nearest representable value. */ -typedef struct ShellState ShellState; -struct ShellState { - sqlite3 *db; /* The database */ - u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ - u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ - u8 nEqpLevel; /* Depth of the EQP output graph */ - u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ - u8 bSafeMode; /* True to prohibit unsafe operations */ - u8 bSafeModePersist; /* The long-term value of bSafeMode */ - u8 eRestoreState; /* See comments above doAutoDetectRestore() */ - unsigned statsOn; /* True to display memory stats before each finalize */ - unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ - u8 nPopOutput; /* Revert .output settings when reaching zero */ - u8 nPopMode; /* Revert .mode settings when reaching zero */ - u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ - int inputNesting; /* Track nesting level of .read and other redirects */ - double prevTimer; /* Last reported timer value */ - double tmProgress; /* --timeout option for .progress */ - i64 lineno; /* Line number of last line read from in */ - const char *zInFile; /* Name of the input file */ - int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ - FILE *in; /* Read commands from this stream */ - FILE *out; /* Write results here */ - FILE *traceOut; /* Output for sqlite3_trace() */ - int nErr; /* Number of errors seen */ - int writableSchema; /* True if PRAGMA writable_schema=ON */ - int nCheck; /* Number of ".check" commands run */ - unsigned nProgress; /* Number of progress callbacks encountered */ - unsigned mxProgress; /* Maximum progress callbacks before failing */ - unsigned flgProgress; /* Flags for the progress callback */ - unsigned shellFlgs; /* Various flags */ - unsigned nTestRun; /* Number of test cases run */ - unsigned nTestErr; /* Number of test cases that failed */ - sqlite3_int64 szMax; /* --maxsize argument to .open */ - char *zDestTable; /* Name of destination table when MODE_Insert */ - char *zTempFile; /* Temporary file that might need deleting */ - char *zErrPrefix; /* Alternative error message prefix */ - char zTestcase[30]; /* Name of current test case */ - char outfile[FILENAME_MAX]; /* Filename for *out */ - sqlite3_stmt *pStmt; /* Current statement if any. */ - FILE *pLog; /* Write log output here */ - Mode mode; /* Current display mode */ - Mode modePrior; /* Backup */ - struct SavedMode { /* Ability to define custom mode configurations */ - char *zTag; /* Name of this saved mode */ - Mode mode; /* The saved mode */ - } *aSavedModes; /* Array of saved .mode settings. system malloc() */ - int nSavedModes; /* Number of saved .mode settings */ - struct AuxDb { /* Storage space for auxiliary database connections */ - sqlite3 *db; /* Connection pointer */ - const char *zDbFilename; /* Filename used to open the connection */ - char *zFreeOnClose; /* Free this memory allocation on close */ -#if defined(SQLITE_ENABLE_SESSION) - int nSession; /* Number of active sessions */ - OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ -#endif - } aAuxDb[5], /* Array of all database connections */ - *pAuxDb; /* Currently active database connection */ - char *zNonce; /* Nonce for temporary safe-mode escapes */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) - ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ -#endif - struct DotCmdLine { /* Info about arguments to a dot-command */ - const char *zOrig; /* Original text of the dot-command */ - char *zCopy; /* Copy of zOrig, from malloc() */ - int nAlloc; /* Size of allocates for arrays below */ - int nArg; /* Number of argument slots actually used */ - char **azArg; /* Pointer to each argument, dequoted */ - int *aiOfst; /* Offset into zOrig[] for start of each arg */ - char *abQuot; /* True if the argment was originally quoted */ - } dot; -#ifdef SQLITE_SHELL_FIDDLE - struct { - const char * zInput; /* Input string from wasm/JS proxy */ - const char * zPos; /* Cursor pos into zInput */ - const char * zDefaultDbName; /* Default name for db file */ - } wasm; -#endif -}; - -#ifdef SQLITE_SHELL_FIDDLE -static ShellState shellState; -#endif - +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_uint64 v = 0; + static const struct { char *zSuffix; unsigned int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + if( v > 0x0fffffffffffffffULL ) goto integer_overflow; + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + if( v>=922337203685477580LL ){ + if( v>922337203685477580LL || zArg[0]>='8' ) goto integer_overflow; + } + v = v*10 + (zArg[0] - '0'); + zArg++; + } + } + for(i=0; i0x7fffffffffffffffULL ) goto integer_overflow; + return isNeg? -(sqlite3_int64)v : (sqlite3_int64)v; +integer_overflow: + return isNeg ? (i64)0x8000000000000000LL : 0x7fffffffffffffffLL; +} -/* Allowed values for ShellState.mode.autoEQP +/* +** A variable length string to which one can append text. */ -#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ -#define AUTOEQP_on 1 /* Automatic EQP is on */ -#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ -#define AUTOEQP_full 3 /* Show full EXPLAIN */ +typedef struct ShellText ShellText; +struct ShellText { + char *zTxt; /* The text */ + i64 n; /* Number of bytes of zTxt[] actually used */ + i64 nAlloc; /* Number of bytes allocated for zTxt[] */ +}; -/* Allowed values for ShellState.openMode +/* +** Initialize and destroy a ShellText object */ -#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ -#define SHELL_OPEN_NORMAL 1 /* Normal database file */ -#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ -#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ -#define SHELL_OPEN_DESERIALIZE 4 /* Open using sqlite3_deserialize() */ -#define SHELL_OPEN_HEXDB 5 /* Use "dbtotxt" output as data source */ +static void initText(ShellText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(ShellText *p){ + sqlite3_free(p->zTxt); + initText(p); +} -/* Allowed values for ShellState.eTraceType +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. */ -#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ -#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ -#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ +static void appendText(ShellText *p, const char *zAppend, char quote){ + i64 len; + i64 i; + i64 nAppend = strlen30(zAppend); -/* Bits in the ShellState.flgProgress variable */ -#define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ -#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progress - ** callback limit is reached, and for each - ** top-level SQL statement */ -#define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ -#define SHELL_PROGRESS_TMOUT 0x08 /* Stop after tmProgress seconds */ + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; izTxt==0 || p->n+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->zTxt = sqlite3_realloc64(p->zTxt, p->nAlloc); + shell_check_oom(p->zTxt); + } -/* -** These are the allowed shellFlgs values -*/ -#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ -#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ -#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ -#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ -#define SHFLG_NoErrLineno 0x00000010 /* Omit line numbers from error msgs */ -#define SHFLG_CountChanges 0x00000020 /* .changes setting */ -#define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ -#define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ -#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ + if( quote ){ + char *zCsr = p->zTxt+p->n; + *zCsr++ = quote; + for(i=0; in = (i64)(zCsr - p->zTxt); + *zCsr = '\0'; + }else{ + memcpy(p->zTxt+p->n, zAppend, nAppend); + p->n += nAppend; + p->zTxt[p->n] = '\0'; + } +} /* -** Macros for testing and setting shellFlgs +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. */ -#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) -#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) -#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) +static char quoteChar(const char *zName){ + int i; + if( zName==0 ) return '"'; + if( !IsAlpha(zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !IsAlnum(zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} /* -** These are the allowed values for Mode.eMode. There is a lot of overlap -** between these values and the Mode.spec.eStyle values, but they are not -** one-to-one, and thus need to be tracked separately. +** Construct a fake object name and column list to describe the structure +** of the view, virtual table, or table valued function zSchema.zName. +** +** The returned string comes from sqlite3_mprintf() and should be freed +** by the caller using sqlite3_free(). */ -#define MODE_Ascii 0 /* Use ASCII unit and record separators (0x1F/0x1E) */ -#define MODE_Box 1 /* Unicode box-drawing characters */ -#define MODE_C 2 /* Comma-separated list of C-strings */ -#define MODE_Column 3 /* One record per line in neat columns */ -#define MODE_Count 4 /* Output only a count of the rows of output */ -#define MODE_Csv 5 /* Quote strings, numbers are plain */ -#define MODE_Html 6 /* Generate an XHTML table */ -#define MODE_Insert 7 /* Generate SQL "insert" statements */ -#define MODE_JAtom 8 /* Comma-separated list of JSON atoms */ -#define MODE_JObject 9 /* One JSON object per row */ -#define MODE_Json 10 /* Output JSON */ -#define MODE_Line 11 /* One column per line. Blank line between records */ -#define MODE_List 12 /* One record per line with a separator */ -#define MODE_Markdown 13 /* Markdown formatting */ -#define MODE_Off 14 /* No query output shown */ -#define MODE_Psql 15 /* Similar to psql */ -#define MODE_QBox 16 /* BOX with SQL-quoted content */ -#define MODE_Quote 17 /* Quote values as for SQL */ -#define MODE_Split 18 /* Split-column mode */ -#define MODE_Table 19 /* MySQL-style table formatting */ -#define MODE_Tabs 20 /* Tab-separated values */ -#define MODE_Tcl 21 /* Space-separated list of TCL strings */ -#define MODE_Www 22 /* Full web-page output */ +static char *shellFakeSchema( + sqlite3 *db, /* The database connection containing the vtab */ + const char *zSchema, /* Schema of the database holding the vtab */ + const char *zName /* The name of the virtual table */ +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + ShellText s; + char cQuote; + char *zDiv = "("; + int nRow = 0; -#define MODE_BUILTIN 22 /* Maximum built-in mode */ -#define MODE_BATCH 50 /* Default mode for batch processing */ -#define MODE_TTY 51 /* Default mode for interactive processing */ -#define MODE_USER 75 /* First user-defined mode */ -#define MODE_N_USER 25 /* Maximum number of user-defined modes */ + zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", + zSchema ? zSchema : "main", zName); + shell_check_oom(zSql); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + initText(&s); + if( zSchema ){ + cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; + appendText(&s, zSchema, cQuote); + appendText(&s, ".", 0); + } + cQuote = quoteChar(zName); + appendText(&s, zName, cQuote); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); + nRow++; + appendText(&s, zDiv, 0); + zDiv = ","; + if( zCol==0 ) zCol = ""; + cQuote = quoteChar(zCol); + appendText(&s, zCol, cQuote); + } + appendText(&s, ")", 0); + sqlite3_finalize(pStmt); + if( nRow==0 ){ + freeText(&s); + s.zTxt = 0; + } + return s.zTxt; +} /* -** Information about built-in display modes +** SQL function: strtod(X) +** +** Use the C-library strtod() function to convert string X into a double. +** Used for comparing the accuracy of SQLite's internal text-to-float conversion +** routines against the C-library. */ -typedef struct ModeInfo ModeInfo; -struct ModeInfo { - char zName[9]; /* Symbolic name of the mode */ - unsigned char eCSep; /* Column separator */ - unsigned char eRSep; /* Row separator */ - unsigned char eNull; /* Null representation */ - unsigned char eText; /* Default text encoding */ - unsigned char eHdr; /* Default header encoding. */ - unsigned char eBlob; /* Default blob encoding. */ - unsigned char bHdr; /* Show headers by default. 0: n/a, 1: no 2: yes */ - unsigned char eStyle; /* Underlying QRF style */ - unsigned char eCx; /* 0: other, 1: line, 2: columnar */ - unsigned char mFlg; /* Flags. 1=border-off 2=split-column */ -}; - -/* String constants used by built-in modes */ -static const char *aModeStr[] = - /* 0 1 2 3 4 5 6 7 8 */ - { 0, "\n", "|", " ", ",", "\r\n", "\036", "\037", "\t", - "", "NULL", "null", "\"\"", ": ", }; - /* 9 10 11 12 13 */ +static void shellStrtod( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + char *z = (char*)sqlite3_value_text(apVal[0]); + UNUSED_PARAMETER(nVal); + if( z==0 ) return; + sqlite3_result_double(pCtx, strtod(z,0)); +} -static const ModeInfo aModeInfo[] = { -/* zName eCSep eRSep eNull eText eHdr eBlob bHdr eStyle eCx mFlg */ - { "ascii", 7, 6, 9, 1, 1, 0, 1, 12, 0, 0 }, - { "box", 0, 0, 9, 1, 1, 0, 2, 1, 2, 0 }, - { "c", 4, 1, 10, 5, 5, 4, 1, 12, 0, 0 }, - { "column", 0, 0, 9, 1, 1, 0, 2, 2, 2, 0 }, - { "count", 0, 0, 0, 0, 0, 0, 0, 3, 0, 0 }, - { "csv", 4, 5, 9, 3, 3, 0, 1, 12, 0, 0 }, - { "html", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 }, - { "insert", 0, 0, 10, 2, 2, 0, 1, 8, 0, 0 }, - { "jatom", 4, 1, 11, 6, 6, 0, 1, 12, 0, 0 }, - { "jobject", 0, 1, 11, 6, 6, 0, 0, 10, 0, 0 }, - { "json", 0, 0, 11, 6, 6, 0, 0, 9, 0, 0 }, - { "line", 13, 1, 9, 1, 1, 0, 0, 11, 1, 0 }, - { "list", 2, 1, 9, 1, 1, 0, 1, 12, 0, 0 }, - { "markdown", 0, 0, 9, 1, 1, 0, 2, 13, 2, 0 }, - { "off", 0, 0, 0, 0, 0, 0, 0, 14, 0, 0 }, - { "psql", 0, 0, 9, 1, 1, 0, 2, 19, 2, 1 }, - { "qbox", 0, 0, 10, 2, 1, 0, 2, 1, 2, 0 }, - { "quote", 4, 1, 10, 2, 2, 0, 1, 12, 0, 0 }, - { "split", 0, 0, 9, 1, 1, 0, 1, 2, 2, 2 }, - { "table", 0, 0, 9, 1, 1, 0, 2, 19, 2, 0 }, - { "tabs", 8, 1, 9, 3, 3, 0, 1, 12, 0, 0 }, - { "tcl", 3, 1, 12, 5, 5, 4, 1, 12, 0, 0 }, - { "www", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 } -}; /* | / / | / / | | \ - ** | / / | / / | | \_ 2: columnar - ** Index into aModeStr[] | / / | | 1: line - ** | / / | | 0: other - ** | / / | \ - ** text encoding |/ | show | \ - ** v-------------------' | hdrs? | The QRF style - ** 0: n/a blob | v-----' - ** 1: plain v_---------' 0: n/a - ** 2: sql 0: auto 1: no - ** 3: csv 1: as-text 2: yes - ** 4: html 2: sql - ** 5: c 3: hex - ** 6: json 4: c - ** 5: json - ** 6: size - ******************************************************************/ /* -** These are the column/row/line separators used by the various -** import/export modes. +** SQL function: dtostr(X) +** +** Use the C-library printf() function to convert real value X into a string. +** Used for comparing the accuracy of SQLite's internal float-to-text conversion +** routines against the C-library. */ -#define SEP_Column "|" -#define SEP_Row "\n" -#define SEP_Tab "\t" -#define SEP_Space " " -#define SEP_Comma "," -#define SEP_CrLf "\r\n" -#define SEP_Unit "\x1F" -#define SEP_Record "\x1E" +static void shellDtostr( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + double r = sqlite3_value_double(apVal[0]); + int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + char z[400]; + if( n<1 ) n = 1; + if( n>350 ) n = 350; + sprintf(z, "%#+.*e", n, r); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); +} /* -** Default values for the various QRF limits +** SQL function: shell_add_schema(S,X) +** +** Add the schema name X to the CREATE statement in S and return the result. +** Examples: +** +** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); +** +** Also works on +** +** CREATE INDEX +** CREATE UNIQUE INDEX +** CREATE VIEW +** CREATE TRIGGER +** CREATE VIRTUAL TABLE +** +** This UDF is used by the .schema command to insert the schema name of +** attached databases into the middle of the sqlite_schema.sql field. */ -#ifndef DFLT_CHAR_LIMIT -# define DFLT_CHAR_LIMIT 300 -#endif -#ifndef DFLT_LINE_LIMIT -# define DFLT_LINE_LIMIT 5 -#endif -#ifndef DFLT_TITLE_LIMIT -# define DFLT_TITLE_LIMIT 20 -#endif +static void shellAddSchemaName( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + static const char *aPrefix[] = { + "TABLE", + "INDEX", + "UNIQUE INDEX", + "VIEW", + "TRIGGER", + "VIRTUAL TABLE" + }; + int i = 0; + const char *zIn = (const char*)sqlite3_value_text(apVal[0]); + const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); + const char *zName = (const char*)sqlite3_value_text(apVal[2]); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + UNUSED_PARAMETER(nVal); + if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ + for(i=0; imode.spec.nLineLimit = DFLT_LINE_LIMIT; p->mode.spec.bTextJsonb = QRF_Yes; p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + p->mode.spec.nMultiInsert = DFLT_MULTI_INSERT; p->mode.mFlags = mFlags; } } @@ -26467,10 +26546,12 @@ static int shell_exec( memcpy(&spec, &pArg->mode.spec, sizeof(spec)); spec.xWrite = shellWriteQR; spec.pWriteArg = (void*)pArg; - if( pArg->mode.eMode==MODE_Insert && ShellHasFlag(pArg, SHFLG_PreserveRowid) ){ + if( pArg->mode.eMode==MODE_Insert && ShellHasFlag(pArg,SHFLG_PreserveRowid) ){ spec.bTitles = QRF_SW_On; } - assert( pArg->mode.eMode>=0 && pArg->mode.eModemode.eMode>=0 &&*/ pArg->mode.eModemode.eMode].eStyle; if( pArg->mode.bAutoScreenWidth ){ spec.nScreenWidth = shellScreenWidth(); @@ -26862,6 +26943,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ p->mode.spec.eText = QRF_TEXT_Sql; p->mode.spec.eBlob = QRF_BLOB_Sql; p->mode.spec.bTitles = QRF_No; + p->mode.spec.nCharLimit = 0; rc = shell_exec(p, sSelect.zTxt, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); @@ -27006,7 +27088,7 @@ static const char *(azHelp[]) = { " --schema SCHEMA Use SCHEMA instead of \"main\"", " --help Show CMD details", ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", - ",headers on|off Turn display of headers on or off", + ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", #ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", @@ -27014,9 +27096,10 @@ static const char *(azHelp[]) = { #ifndef SQLITE_OMIT_TEST_CONTROL ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif - ".indexes ?TABLE? Show names of indexes", - " If TABLE is specified, only show indexes for", - " tables matching TABLE using the LIKE operator.", + ".indexes ?PATTERN? Show names of indexes matching PATTERN", + " -a|--all Also show system-generated indexes", + " --expr Show only expression indexes", + " --sys Show only system-generated indexes", ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", @@ -27254,6 +27337,8 @@ static const struct { " can also be \"off\" to mean \"0,0,0\" or \"on\" to\n" " mean \"5,300,20\".\n" " --list List available modes\n" +" --multiinsert N In \"insert\" mode, put multiple rows on a single\n" +" INSERT statement until the size exceeds N bytes.\n" " --null STRING Render SQL NULL values as the given string\n" " --once Setting changes to the right are reverted after\n" " the next SQL command.\n" @@ -30752,7 +30837,7 @@ static int outputDumpWarning(ShellState *p, const char *zLike){ sqlite3_stmt *pStmt = 0; shellPreparePrintf(p->db, &rc, &pStmt, "SELECT 1 FROM sqlite_schema o WHERE " - "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" + "sql LIKE 'CREATE VIRTUAL TABLE%%' AND (%s)", zLike ? zLike : "true" ); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ cli_puts("/* WARNING: " @@ -31278,9 +31363,9 @@ static int modeTitleDsply(ShellState *p, int bAll){ /* Variable "v" is the truth table that will determine the answer ** ** Actual encoding is different from default - ** vvvvvvvv */ - sqlite3_uint64 v = 0x0133013311220102; - /* ^^^^ ^^^^ + ** vvvvvvvv */ + sqlite3_uint64 v = UINT64_C(0x0133013311220102); + /* ^^^^ ^^^^ ** Upper 2-byte groups for when ON/OFF disagrees with ** the default. */ @@ -31332,6 +31417,8 @@ static int modeTitleDsply(ShellState *p, int bAll){ ** can also be "off" to mean "0,0,0" or "on" to ** mean "5,300,20". ** --list List available modes +** --multiinsert N In "insert" mode, put multiple rows on a single +** INSERT statement until the size exceeds N bytes. ** --null STRING Render SQL NULL values as the given string ** --once Setting changes to the right are reverted after ** the next SQL command. @@ -31447,17 +31534,19 @@ static int dotCmdMode(ShellState *p){ p->mode.spec.bBorder = k & 0x3; } chng = 1; - }else if( 0<=(k=pickStr(z,0,"-charlimit","-linelimit","-titlelimit","")) ){ - int w; /* 0 1 */ + }else if( 0<=(k=pickStr(z,0, + "-charlimit","-linelimit","-titlelimit","-multiinsert","")) ){ + int w; /* 0 1 2 3 */ if( i+1>=nArg ){ dotCmdError(p, i, "missing argument", 0); return 1; } w = integerValue(azArg[++i]); switch( k ){ - case 0: p->mode.spec.nCharLimit = w; break; - case 1: p->mode.spec.nLineLimit = w; break; - default: p->mode.spec.nTitleLimit = w; break; + case 0: p->mode.spec.nCharLimit = w; break; + case 1: p->mode.spec.nLineLimit = w; break; + case 2: p->mode.spec.nTitleLimit = w; break; + default: p->mode.spec.nMultiInsert = w; break; } chng = 1; }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){ @@ -31800,6 +31889,12 @@ static int dotCmdMode(ShellState *p){ p->mode.spec.nTitleLimit); } } + if( bAll + || (p->mode.spec.nMultiInsert && p->mode.spec.eStyle==QRF_STYLE_Insert) + ){ + sqlite3_str_appendf(pDesc, " --multiinsert %u", + p->mode.spec.nMultiInsert); + } zSetting = aModeStr[pI->eNull]; if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){ sqlite3_str_appendf(pDesc, " --null "); @@ -31961,8 +32056,7 @@ static int dotCmdOutput(ShellState *p){ zBom = zBomUtf8; }else if( cli_strcmp(z,"-plain")==0 ){ bPlain = 1; - }else if( c=='o' && z[0]=='1' && z[1]!=0 && z[2]==0 - && (z[1]=='x' || z[1]=='e' || z[1]=='w') ){ + }else if( c=='o' && sqlite3_strglob("-[ewx]",z)==0 ){ if( bKeep || eMode ){ dotCmdError(p, i, "incompatible with prior options",0); goto dotCmdOutput_error; @@ -33266,6 +33360,92 @@ static int do_meta_command(const char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + if( c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 + || cli_strncmp(azArg[0], "indexes", n)==0) + ){ + sqlite3_str *pSql; + int i; + int allFlag = 0; + int sysFlag = 0; + int exprFlag = 0; + int debugFlag = 0; /* Undocument --debug flag */ + const char *zPattern = 0; + const char *zSep = "WHERE"; + + for(i=1; idb); + sqlite3_str_appendf(pSql, + "SELECT if(t.schema='main',i.name,t.schema||'.'||i.name)\n" + "FROM pragma_table_list t, pragma_index_list(t.name,t.schema) i\n" + ); + if( exprFlag ){ + allFlag = 0; + sqlite3_str_appendf(pSql, + "%s (EXISTS(SELECT 1 FROM pragma_index_xinfo(i.name) WHERE cid=-2)\n" + " OR\n" + " EXISTS(SELECT cid FROM pragma_table_xinfo(t.name) WHERE hidden=2" + " INTERSECT " + " SELECT cid FROM pragma_index_info(i.name)))\n", zSep); + zSep = "AND"; + } + if( sysFlag ){ + sqlite3_str_appendf(pSql, + "%s i.name LIKE 'sqlite__autoindex__%%' ESCAPE '_'\n", zSep); + zSep = "AND"; + }else if( !allFlag ){ + sqlite3_str_appendf(pSql, + "%s i.name NOT LIKE 'sqlite__%%' ESCAPE '_'\n", zSep); + zSep = "AND"; + } + if( zPattern ){ + sqlite3_str_appendf(pSql, "%s i.name LIKE '%%%q%%'\n", zSep, zPattern); + } + sqlite3_str_appendf(pSql, "ORDER BY 1"); + + /* Run the SQL statement in "split" mode. */ + if( debugFlag ){ + cli_printf(stdout,"%s;\n", sqlite3_str_value(pSql)); + }else{ + modePush(p); + modeChange(p, MODE_Split); + shell_exec(p, sqlite3_str_value(pSql), 0); + modePop(p); + } + sqlite3_str_free(pSql); + }else + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ i64 iArg = 0; if( nArg==2 ){ @@ -34643,10 +34823,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } }else - if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) - || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 - || cli_strncmp(azArg[0], "indexes", n)==0) ) - ){ + if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) ){ sqlite3_stmt *pStmt; sqlite3_str *pSql; const char *zPattern = nArg>1 ? azArg[1] : 0; @@ -34658,15 +34835,6 @@ static int do_meta_command(const char *zLine, ShellState *p){ return shellDatabaseError(p->db); } - if( nArg>2 && c=='i' ){ - /* It is an historical accident that the .indexes command shows an error - ** when called with the wrong number of arguments whereas the .tables - ** command does not. */ - eputz("Usage: .indexes ?LIKE-PATTERN?\n"); - rc = 1; - sqlite3_finalize(pStmt); - goto meta_command_exit; - } pSql = sqlite3_str_new(p->db); while( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); @@ -34680,19 +34848,12 @@ static int do_meta_command(const char *zLine, ShellState *p){ sqlite3_str_appendf(pSql, "SELECT %Q||'.'||name FROM ", zDbName); } sqlite3_str_appendf(pSql, "\"%w\".sqlite_schema", zDbName); - if( c=='t' ){ - sqlite3_str_appendf(pSql, - " WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite__%%' ESCAPE '_'" - ); - if( zPattern ){ - sqlite3_str_appendf(pSql," AND name LIKE %Q", zPattern); - } - }else{ - sqlite3_str_appendf(pSql, " WHERE type='index'"); - if( zPattern ){ - sqlite3_str_appendf(pSql," AND tbl_name LIKE %Q", zPattern); - } + sqlite3_str_appendf(pSql, + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_'" + ); + if( zPattern ){ + sqlite3_str_appendf(pSql," AND name LIKE %Q", zPattern); } } rc = sqlite3_finalize(pStmt); @@ -35624,7 +35785,12 @@ static int doAutoDetectRestore(ShellState *p, const char *zSql){ /* ** Run a single line of SQL. Return the number of errors. */ -static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ +static int runOneSqlLine( + ShellState *p, /* Execution context */ + char *zSql, /* SQL to be run */ + const char *zFilename, /* Source file of the sql */ + int startline /* linenumber */ +){ int rc; char *zErrMsg = 0; @@ -35635,7 +35801,9 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ rc = shell_exec(p, zSql, &zErrMsg); END_TIMER(p); if( rc || zErrMsg ){ - char zPrefix[100]; + char zPrefix[2048 + /* must be relatively large: + ** https://sqlite.org/forum/forumpost/205f73db1b2806f5 */]; const char *zErrorTail; const char *zErrorType; if( zErrMsg==0 ){ @@ -35651,9 +35819,17 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ zErrorType = "Error"; zErrorTail = zErrMsg; } - if( in!=0 || !stdin_is_interactive ){ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, - "%s near line %d:", zErrorType, startline); + if( zFilename || !stdin_is_interactive ){ + if( cli_strcmp(zFilename,"cmdline")==0 ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s in %r command line argument:", zErrorType, startline); + }else if( cli_strcmp(zFilename,"")==0 ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d:", zErrorType, startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d of %s:", zErrorType, startline, zFilename); + } }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } @@ -35686,12 +35862,13 @@ static void echo_group_input(ShellState *p, const char *zDo){ ** impl because we need the global shellState and cannot access it from that ** function without moving lots of code around (creating a larger/messier diff). */ -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ +static char *one_input_line(ShellState *p, char *zPrior, int isContinuation){ /* Parse the next line from shellState.wasm.zInput. */ const char *zBegin = shellState.wasm.zPos; const char *z = zBegin; char *zLine = 0; i64 nZ = 0; + FILE *in = p->in; UNUSED_PARAMETER(in); UNUSED_PARAMETER(isContinuation); @@ -35749,7 +35926,7 @@ static int process_input(ShellState *p, const char *zSrc){ CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); - zLine = one_input_line(p->in, zLine, nSql>0); + zLine = one_input_line(p, zLine, nSql>0); if( zLine==0 ){ /* End of input */ if( p->in==0 && stdin_is_interactive ) cli_puts("\n", p->out); @@ -35816,7 +35993,7 @@ static int process_input(ShellState *p, const char *zSrc){ break; }else if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); - errCnt += runOneSqlLine(p, zSql, p->in, startline); + errCnt += runOneSqlLine(p, zSql, p->zInFile, startline); CONTINUE_PROMPT_RESET; nSql = 0; if( p->nPopOutput ){ @@ -35840,7 +36017,7 @@ static int process_input(ShellState *p, const char *zSrc){ if( nSql ){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); - errCnt += runOneSqlLine(p, zSql, p->in, startline); + errCnt += runOneSqlLine(p, zSql, p->zInFile, startline); CONTINUE_PROMPT_RESET; } free(zSql); @@ -36179,6 +36356,22 @@ static int vfstraceOut(const char *z, void *pArg){ return 1; } +#if defined(SQLITE_DEBUG) && !defined(SQLITE_SHELL_FIDDLE) +/* Ensure that sqlite3_reset_auto_extension() clears auto-extension +** memory. https://sqlite.org/forum/forumpost/310cb231b07c80d1. +** Testing this: if we (inadvertently) remove the +** sqlite3_reset_auto_extension() call from main(), most shell tests +** will fail because of a leak message in their output. */ +static int auto_ext_leak_tester( + sqlite3 *db, + char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + (void)db; (void)pzErrMsg; (void)pThunk; + return SQLITE_OK; +} +#endif + /* Alternative name to the entry point for Fiddle */ #ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main @@ -36188,7 +36381,7 @@ static int vfstraceOut(const char *z, void *pArg){ ** UTF8, then invoke the traditional main() entry point which is ** renamed using a #define to utf8_main() . */ -#if defined(_WIN32) && !defined(main) +#if defined(_WIN32) && !defined(__MINGW32__) && !defined(main) # define main utf8_main /* Rename entry point to utf_main() */ int SQLITE_CDECL utf8_main(int,char**); /* Forward declaration */ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ @@ -36237,7 +36430,6 @@ int SQLITE_CDECL main(int argc, char **argv){ #ifdef SQLITE_DEBUG sqlite3_int64 mem_main_enter = 0; #endif - char *zErrMsg = 0; #ifdef SQLITE_SHELL_FIDDLE # define data shellState #else @@ -36582,6 +36774,9 @@ int SQLITE_CDECL main(int argc, char **argv){ } #ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); +#ifdef SQLITE_DEBUG + sqlite3_auto_extension( (void (*)(void))auto_ext_leak_tester ); +#endif #endif modeDefault(&data); @@ -36776,15 +36971,7 @@ int SQLITE_CDECL main(int argc, char **argv){ goto shell_main_exit; } }else{ - open_db(&data, 0); - rc = shell_exec(&data, z, &zErrMsg); - if( zErrMsg!=0 ){ - shellEmitError(zErrMsg); - sqlite3_free(zErrMsg); - if( !rc ) rc = 1; - }else if( rc!=0 ){ - cli_printf(stderr,"Error: unable to process SQL \"%s\"\n", z); - } + rc = runOneSqlLine(&data, z, "cmdline", i); if( bail_on_error ) goto shell_main_exit; } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) @@ -36850,25 +37037,17 @@ int SQLITE_CDECL main(int argc, char **argv){ goto shell_main_exit; } }else{ - open_db(&data, 0); - rc = shell_exec(&data, azCmd[i], &zErrMsg); - if( zErrMsg || rc ){ - if( zErrMsg!=0 ){ - shellEmitError(zErrMsg); - }else{ - cli_printf(stderr, - "Error: unable to process SQL: %s\n", azCmd[i]); - } - sqlite3_free(zErrMsg); - if( rc==0 ) rc = 1; - goto shell_main_exit; - } + rc = runOneSqlLine(&data, azCmd[i], "cmdline", aiCmd[i]); if( data.nPopMode ){ modePop(&data); data.nPopMode = 0; } + if( rc ){ + goto shell_main_exit; + } + } - if( data.nPopOutput ){ + if( data.nPopOutput && azCmd[i][0]!='.' ){ output_reset(&data); data.nPopOutput = 0; }else{ @@ -36979,6 +37158,7 @@ int SQLITE_CDECL main(int argc, char **argv){ if( bEnableVfstrace ){ vfstrace_unregister("trace"); } + sqlite3_reset_auto_extension(); #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ cli_printf(stderr,"Memory leaked: %u bytes\n", diff --git a/cgosqlite/sqlite3.c b/cgosqlite/sqlite3.c index e137809..9f8024c 100644 --- a/cgosqlite/sqlite3.c +++ b/cgosqlite/sqlite3.c @@ -2,7 +2,7 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.52.0. By combining all the individual C code files into this +** version 3.53.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -20,7 +20,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** 557aeb43869d3585137b17690cb3b64f7de6 with changes in files: +** c88b22011a54b4f6fbd149e9f8e4de77658c with changes in files: ** ** */ @@ -469,12 +469,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.52.0" -#define SQLITE_VERSION_NUMBER 3052000 -#define SQLITE_SOURCE_ID "2026-03-06 16:01:44 557aeb43869d3585137b17690cb3b64f7de6921774daae9e56403c3717dceab6" -#define SQLITE_SCM_BRANCH "trunk" -#define SQLITE_SCM_TAGS "release major-release version-3.52.0" -#define SQLITE_SCM_DATETIME "2026-03-06T16:01:44.367Z" +#define SQLITE_VERSION "3.53.1" +#define SQLITE_VERSION_NUMBER 3053001 +#define SQLITE_SOURCE_ID "2026-05-05 10:34:17 c88b22011a54b4f6fbd149e9f8e4de77658ce58143a1af0e3785e4e6475127e9" +#define SQLITE_SCM_BRANCH "branch-3.53" +#define SQLITE_SCM_TAGS "release version-3.53.1" +#define SQLITE_SCM_DATETIME "2026-05-05T10:34:17.344Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -901,7 +901,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal only */ /* ** CAPI3REF: Flags For File Open Operations @@ -1613,6 +1613,12 @@ struct sqlite3_io_methods { #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO +/* reserved file-control numbers: +** 101 +** 102 +** 103 +*/ + /* ** CAPI3REF: Mutex Handle @@ -2034,7 +2040,8 @@ SQLITE_API int sqlite3_os_end(void); ** are called "anytime configuration options". ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before ** [sqlite3_shutdown()] with a first argument that is not an anytime -** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** configuration option, then the sqlite3_config() call will +** return SQLITE_MISUSE. ** Note, however, that ^sqlite3_config() can be called as part of the ** implementation of an application-defined [sqlite3_os_init()]. ** @@ -2600,9 +2607,10 @@ struct sqlite3_mem_methods { ** is less than 8. The "sz" argument should be a multiple of 8 less than ** 65536. If "sz" does not meet this constraint, it is reduced in size until ** it does. -**

  • The third argument ("cnt") is the number of slots. Lookaside is disabled -** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so -** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +**

  • The third argument ("cnt") is the number of slots. +** Lookaside is disabled if "cnt"is less than 1. +* The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" ** parameter is usually chosen so that the product of "sz" and "cnt" is less ** than 1,000,000. ** @@ -2981,7 +2989,7 @@ struct sqlite3_mem_methods { ** This option takes two arguments which are an integer and a pointer ** to an integer. The first argument is a small integer, between 3 and 23, or ** zero. The FP_DIGITS setting is changed to that small integer, or left -** altered if the first argument is zero or out of range. The second argument +** unaltered if the first argument is zero or out of range. The second argument ** is a pointer to an integer. If the pointer is not NULL, then the value of ** the FP_DIGITS setting, after possibly being modified by the first ** arguments, is written into the integer to which the second argument points. @@ -2993,10 +3001,12 @@ struct sqlite3_mem_methods { ** **

    Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is an integer. -** The second argument is a pointer to an integer. If the first argument is 1, -** then the option becomes enabled. If the first integer argument is 0, then the -** option is disabled. If the first argument is -1, then the option setting +** The first argument (the third parameter to sqlite3_db_config()) is +** an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, +** then the option is disabled. +** If the first argument is -1, then the option setting ** is unchanged. The second argument, the pointer to an integer, may be NULL. ** If the second argument is not NULL, then a value of 0 or 1 is written into ** the integer to which the second argument points, depending on whether the @@ -4811,7 +4821,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. ** [sqlite3_prepare_v3()] has an extra -** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is some times +** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is sometimes ** needed for special purpose or to pass along security restrictions. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently @@ -6173,8 +6183,9 @@ SQLITE_API int sqlite3_create_window_function( ** of the strings must be aligned to a 2-byte boundary. ** ** [[SQLITE_UTF8_ZT]]

    SQLITE_UTF8_ZT
    This option can only be -** used to specify the text encoding to strings input to [sqlite3_result_text64()] -** and [sqlite3_bind_text64()]. It means that the input string (call it "z") +** used to specify the text encoding to strings input to +** [sqlite3_result_text64()] and [sqlite3_bind_text64()]. +** The SQLITE_UTF8_ZT encoding means that the input string (call it "z") ** is UTF-8 encoded and that it is zero-terminated. If the length parameter ** (call it "n") is non-negative, this encoding option means that the caller ** guarantees that z array contains at least n+1 bytes and that the z[n] @@ -6426,26 +6437,22 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** the SQL function that supplied the [sqlite3_value*] parameters. ** ** As long as the input parameter is correct, these routines can only -** fail if an out-of-memory error occurs during a format conversion. -** Only the following subset of interfaces are subject to out-of-memory -** errors: -** -**
      -**
    • sqlite3_value_blob() -**
    • sqlite3_value_text() -**
    • sqlite3_value_text16() -**
    • sqlite3_value_text16le() -**
    • sqlite3_value_text16be() -**
    • sqlite3_value_bytes() -**
    • sqlite3_value_bytes16() -**
    -** +** fail if an out-of-memory error occurs while trying to do a +** UTF8→UTF16 or UTF16→UTF8 conversion. ** If an out-of-memory error occurs, then the return value from these ** routines is the same as if the column had contained an SQL NULL value. -** Valid SQL NULL returns can be distinguished from out-of-memory errors -** by invoking the [sqlite3_errcode()] immediately after the suspect +** If the input sqlite3_value was not obtained from [sqlite3_value_dup()], +** then valid SQL NULL returns can also be distinguished from +** out-of-memory errors after extracting the value +** by invoking the [sqlite3_errcode()] immediately after the suspicious ** return value is obtained and before any ** other SQLite interface is called on the same [database connection]. +** If the input sqlite3_value was obtained from sqlite3_value_dup() then +** it is disconnected from the database connection and so sqlite3_errcode() +** will not work. +** In that case, the only way to distinguish an out-of-memory +** condition from a true SQL NULL is to invoke sqlite3_value_type() on the +** input to see if it is NULL prior to trying to extract the value. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -6472,7 +6479,8 @@ SQLITE_API int sqlite3_value_frombind(sqlite3_value*); ** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) ** returns something other than SQLITE_TEXT, then the return value from ** sqlite3_value_encoding(X) is meaningless. ^Calls to -** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], +** [sqlite3_value_text16be(X)], ** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or ** [sqlite3_value_bytes16(X)] might change the encoding of the value X and ** thus change the return from subsequent calls to sqlite3_value_encoding(X). @@ -6603,17 +6611,17 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** query execution, under some circumstances the associated auxiliary data ** might be preserved. An example of where this might be useful is in a ** regular-expression matching function. The compiled version of the regular -** expression can be stored as auxiliary data associated with the pattern string. -** Then as long as the pattern string remains the same, +** expression can be stored as auxiliary data associated with the pattern +** string. Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data -** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument -** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no auxiliary data -** associated with the function argument, the sqlite3_get_auxdata(C,N) interface -** returns a NULL pointer. +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary +** data associated by the sqlite3_set_auxdata(C,N,P,X) function with the +** Nth argument value to the application-defined function. ^N is zero +** for the left-most function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) +** interface returns a NULL pointer. ** ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the ** N-th argument of the application-defined function. ^Subsequent @@ -6818,7 +6826,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], ** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that ** the result text is both UTF-8 and zero-terminated. In other words, -** SQLITE_UTF8_ZT means that the Z array holds at least N+1 byes and that +** SQLITE_UTF8_ZT means that the Z array holds at least N+1 bytes and that ** the Z[N] is zero. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. @@ -7930,7 +7938,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** 
     **    int xEntryPoint(
 **      sqlite3 *db,
-**      const char **pzErrMsg,
+**      char **pzErrMsg,
 **      const struct sqlite3_api_routines *pThunk
 **    );
 **
    )^ @@ -8680,13 +8688,6 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor -** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex -** implementation is included with the library. In this case the -** application must supply a custom mutex implementation using the -** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function -** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() @@ -9041,6 +9042,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_ATOF 34 #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* @@ -9188,7 +9190,7 @@ SQLITE_API void sqlite3_str_free(sqlite3_str*); ** inside [sqlite3_str] object X back to zero bytes in length. ** ** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string -** under construction to be N bytes are less. This routine is a no-op if +** under construction to be N bytes or less. This routine is a no-op if ** N is negative or if the string is already N bytes or smaller in size. ** ** These methods do not return a result code. ^If an error occurs, that fact @@ -10732,7 +10734,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** sqlite3_vtab_distinct() return value ** Rows are returned in aOrderBy order -** Rows with the same value in all aOrderBy columns are adjacent +** Rows with the same value in all aOrderBy columns are +** adjacent ** Duplicates over all colUsed columns may be omitted ** 0yesyesno ** 1noyesno @@ -10741,8 +10744,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** ** ^For the purposes of comparing virtual table output values to see if the -** values are the same value for sorting purposes, two NULL values are considered -** to be the same. In other words, the comparison operator is "IS" +** values are the same value for sorting purposes, two NULL values are +** considered to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** ** If a virtual table implementation is unable to meet the requirements @@ -11052,7 +11055,7 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] and the -** [nexec and ncycle] columnes of the [bytecode virtual table]. +** [nexec and ncycle] columns of the [bytecode virtual table]. */ SQLITE_API int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -11596,29 +11599,30 @@ SQLITE_API int sqlite3_deserialize( ** ** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to ** parameter that is the first argument of the [carray() table-valued function]. -** The S parameter is a pointer to the [prepared statement] that uses the carray() -** functions. I is the parameter index to be bound. I must be the index of the -** parameter that is the first argument to the carray() table-valued function. -** P is a pointer to the array to be bound, and N is the number of elements in -** the array. The F argument is one of constants [SQLITE_CARRAY_INT32], -** [SQLITE_CARRAY_INT64], [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], +** The S parameter is a pointer to the [prepared statement] that uses the +** carray() functions. I is the parameter index to be bound. I must be the +** index of the parameter that is the first argument to the carray() +** table-valued function. P is a pointer to the array to be bound, and N +** is the number of elements in the array. The F argument is one of +** constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], ** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. ** ** If the X argument is not a NULL pointer or one of the special ** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke ** the function X with argument D when it is finished using the data in P. ** The call to X(D) is a destructor for the array P. The destructor X(D) -** is invoked even if the call to sqlite3_carray_bind() fails. If the X +** is invoked even if the call to sqlite3_carray_bind_v2() fails. If the X ** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes ** that the data static and the destructor is never invoked. If the X ** parameter is the special-case value [SQLITE_TRANSIENT], then ** sqlite3_carray_bind_v2() makes its own private copy of the data prior ** to returning and never invokes the destructor X. ** -** The sqlite3_carray_bind() function works the same as sqlite_carray_bind_v2() +** The sqlite3_carray_bind() function works the same as sqlite3_carray_bind_v2() ** with a D parameter set to P. In other words, ** sqlite3_carray_bind(S,I,P,N,F,X) is same as -** sqlite3_carray_bind(S,I,P,N,F,X,P). +** sqlite3_carray_bind_v2(S,I,P,N,F,X,P). */ SQLITE_API int sqlite3_carray_bind_v2( sqlite3_stmt *pStmt, /* Statement to be bound */ @@ -13659,6 +13663,232 @@ SQLITE_API int sqlite3session_config(int op, void *pArg); */ #define SQLITE_SESSION_CONFIG_STRMSIZE 1 +/* +** CAPI3REF: Configure a changegroup object +** +** Configure the changegroup object passed as the first argument. +** At present the only valid value for the second parameter is +** [SQLITE_CHANGEGROUP_CONFIG_PATCHSET]. +*/ +SQLITE_API int sqlite3changegroup_config(sqlite3_changegroup*, int, void *pArg); + +/* +** CAPI3REF: Options for sqlite3changegroup_config(). +** +** The following values may be passed as the 2nd parameter to +** sqlite3changegroup_config(). +** +**
    SQLITE_CHANGEGROUP_CONFIG_PATCHSET
    +** A changegroup object generates either a changeset or patchset. Usually, +** this is determined by whether the first call to sqlite3changegroup_add() +** is passed a changeset or a patchset. Or, if the first changes are added +** to the changegroup object using the sqlite3changegroup_change_xxx() +** APIs, then this option may be used to configure whether the changegroup +** object generates a changeset or patchset. +** +** When this option is invoked, parameter pArg must point to a value of +** type int. If the changegroup currently contains zero changes, and the +** value of the int variable is zero or greater than zero, then the +** changegroup is configured to generate a changeset or patchset, +** respectively. It is a no-op, not an error, if the changegroup is not +** configured because it has already started accumulating changes. +** +** Before returning, the int variable is set to 0 if the changegroup is +** configured to generate a changeset, or 1 if it is configured to generate +** a patchset. +*/ +#define SQLITE_CHANGEGROUP_CONFIG_PATCHSET 1 + + +/* +** CAPI3REF: Begin adding a change to a changegroup +** +** This API is used, in concert with other sqlite3changegroup_change_xxx() +** APIs, to add changes to a changegroup object one at a time. To add a +** single change, the caller must: +** +** 1. Invoke sqlite3changegroup_change_begin() to indicate the type of +** change (INSERT, UPDATE or DELETE), the affected table and whether +** or not the change should be marked as indirect. +** +** 2. Invoke sqlite3changegroup_change_int64() or one of the other four +** value functions - _null(), _double(), _text() or _blob() - one or +** more times to specify old.* and new.* values for the change being +** constructed. +** +** 3. Invoke sqlite3changegroup_change_finish() to either finish adding +** the change to the group, or to discard the change altogether. +** +** The first argument to this function must be a pointer to the existing +** changegroup object that the change will be added to. The second argument +** must be SQLITE_INSERT, SQLITE_UPDATE or SQLITE_DELETE. The third is the +** name of the table that the change affects, and the fourth is a boolean +** flag specifying whether the change should be marked as "indirect" (if +** bIndirect is non-zero) or not indirect (if bIndirect is zero). +** +** Following a successful call to this function, this function may not be +** called again on the same changegroup object until after +** sqlite3changegroup_change_finish() has been called. Doing so is an +** SQLITE_MISUSE error. +** +** The changegroup object passed as the first argument must be already +** configured with schema data for the specified table. It may be configured +** either by calling sqlite3changegroup_schema() with a database that contains +** the table, or sqlite3changegroup_add() with a changeset that contains the +** table. If the changegroup object has not been configured with a schema for +** the specified table when this function is called, SQLITE_ERROR is returned. +** +** If successful, SQLITE_OK is returned. Otherwise, if an error occurs, an +** SQLite error code is returned. In this case, if argument pzErr is non-NULL, +** then (*pzErr) may be set to point to a buffer containing a utf-8 formated, +** nul-terminated, English language error message. It is the responsibility +** of the caller to eventually free this buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_begin( + sqlite3_changegroup*, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +); + +/* +** CAPI3REF: Add a 64-bit integer to a changegroup +** +** This function may only be called between a successful call to +** sqlite3changegroup_change_begin() and its matching +** sqlite3changegroup_change_finish() call. If it is called at any +** other time, it is an SQLITE_MISUSE error. Calling this function +** specifies a 64-bit integer value to be used in the change currently being +** added to the changegroup object passed as the first argument. +** +** The second parameter, bNew, specifies whether the value is to be part of +** the new.* (if bNew is non-zero) or old.* (if bNew is zero) record of +** the change under construction. If this does not match the type of change +** specified by the preceding call to sqlite3changegroup_change_begin() (i.e. +** an old.* value for an SQLITE_INSERT change, or a new.* value for an +** SQLITE_DELETE), then SQLITE_ERROR is returned. +** +** The third parameter specifies the column of the old.* or new.* record that +** the value will be a part of. If the specified table has an explicit primary +** key, then this is the index of the table column, numbered from 0 in the order +** specified within the CREATE TABLE statement. Or, if the table uses an +** implicit rowid key, then the column 0 is the rowid and the explicit columns +** are numbered starting from 1. If the iCol parameter is less than 0 or greater +** than the index of the last column in the table, SQLITE_RANGE is returned. +** +** The fourth parameter is the integer value to use as part of the old.* or +** new.* record. +** +** If this call is successful, SQLITE_OK is returned. Otherwise, if an +** error occurs, an SQLite error code is returned. +*/ +SQLITE_API int sqlite3changegroup_change_int64( + sqlite3_changegroup*, + int bNew, + int iCol, + sqlite3_int64 iVal +); + +/* +** CAPI3REF: Add a NULL to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently under construction with a NULL value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_null(sqlite3_changegroup*, int, int); + +/* +** CAPI3REF: Add an double to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently being constructed with a real value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_double(sqlite3_changegroup*, int, int, double); + +/* +** CAPI3REF: Add a text value to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a text value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the text encoded using +** utf-8. Parameter nVal may either be the size of the text value in bytes, or +** else a negative value, in which case the buffer pVal points to is assumed to +** be nul-terminated. +*/ +SQLITE_API int sqlite3changegroup_change_text( + sqlite3_changegroup*, int, int, const char *pVal, int nVal +); + +/* +** CAPI3REF: Add a blob to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a blob value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the blob. Parameter +** nVal is the size of the blob in bytes. +*/ +SQLITE_API int sqlite3changegroup_change_blob( + sqlite3_changegroup*, int, int, const void *pVal, int nVal +); + +/* +** CAPI3REF: Finish adding one-at-at-time changes to a changegroup +** +** This function may only be called following a successful call to +** sqlite3changegroup_change_begin(). Otherwise, it is an SQLITE_MISUSE error. +** +** If parameter bDiscard is non-zero, then the current change is simply +** discarded. In this case this function is always successful and SQLITE_OK +** returned. +** +** If parameter bDiscard is zero, then an attempt is made to add the current +** change to the changegroup. Assuming the changegroup is configured to +** produce a changeset (not a patchset), this requires that: +** +** * If the change is an INSERT or DELETE, then a value must be specified +** for all columns of the new.* or old.* record, respectively. +** +** * If the change is an UPDATE record, then values must be provided for +** the PRIMARY KEY columns of the old.* record, but must not be provided +** for PRIMARY KEY columns of the new.* record. +** +** * If the change is an UPDATE record, then for each non-PRIMARY KEY +** column in the old.* record for which a value has been provided, a +** value must also be provided for the same column in the new.* record. +** Similarly, for each non-PK column in the old.* record for which +** a value is not provided, a value must not be provided for the same +** column in the new.* record. +** +** * All values specified for PRIMARY KEY columns must be non-NULL. +** +** Otherwise, it is an error. +** +** If the changegroup already contains a change for the same row (identified +** by PRIMARY KEY columns), then the current change is combined with the +** existing change in the same way as for sqlite3changegroup_add(). +** +** For a patchset, all of the above rules apply except that it doesn't matter +** whether or not values are provided for the non-PK old.* record columns +** for an UPDATE or DELETE change. This means that code used to produce +** a changeset using the sqlite3changegroup_change_xxx() APIs may also +** be used to produce patchsets. +** +** If the call is successful, SQLITE_OK is returned. Otherwise, if an error +** occurs, an SQLite error code is returned. If an error is returned and +** parameter pzErr is not NULL, then (*pzErr) may be set to point to a buffer +** containing a nul-terminated, utf-8 encoded, English language error message. +** It is the responsibility of the caller to eventually free any such error +** message buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_finish( + sqlite3_changegroup*, + int bDiscard, + char **pzErr +); + /* ** Make sure we can call this stuff from C++. */ @@ -14680,6 +14910,10 @@ struct fts5_api { # undef SQLITE_MAX_DEFAULT_PAGE_SIZE # define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE #endif +#if SQLITE_MAX_DEFAULT_PAGE_SIZE -#endif #ifdef HAVE_INTTYPES_H #include #endif @@ -17319,6 +17551,7 @@ struct VdbeOp { SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ Table *pTab; /* Used when p4type is P4_TABLE */ SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ + Index *pIdx; /* Used when p4type is P4_INDEX */ #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif @@ -17373,20 +17606,21 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT32 (-3) /* P4 is a 32-bit signed integer */ #define P4_SUBPROGRAM (-4) /* P4 is a pointer to a SubProgram structure */ #define P4_TABLE (-5) /* P4 is a pointer to a Table structure */ +#define P4_INDEX (-6) /* P4 is a pointer to an Index structure */ /* Above do not own any resources. Must free those below */ -#define P4_FREE_IF_LE (-6) -#define P4_DYNAMIC (-6) /* Pointer to memory from sqliteMalloc() */ -#define P4_FUNCDEF (-7) /* P4 is a pointer to a FuncDef structure */ -#define P4_KEYINFO (-8) /* P4 is a pointer to a KeyInfo structure */ -#define P4_EXPR (-9) /* P4 is a pointer to an Expr tree */ -#define P4_MEM (-10) /* P4 is a pointer to a Mem* structure */ -#define P4_VTAB (-11) /* P4 is a pointer to an sqlite3_vtab structure */ -#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ -#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ -#define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ -#define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ -#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ -#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ +#define P4_FREE_IF_LE (-7) +#define P4_DYNAMIC (-7) /* Pointer to memory from sqliteMalloc() */ +#define P4_FUNCDEF (-8) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-9) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-10) /* P4 is a pointer to an Expr tree */ +#define P4_MEM (-11) /* P4 is a pointer to a Mem* structure */ +#define P4_VTAB (-12) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_REAL (-13) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-14) /* P4 is a 64-bit signed integer */ +#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-16) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-17) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-18) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -17475,10 +17709,10 @@ typedef struct VdbeOpList VdbeOpList; #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ #define OP_IdxLT 45 /* jump, synopsis: key=r[P3@P4] */ #define OP_IdxGE 46 /* jump, synopsis: key=r[P3@P4] */ -#define OP_RowSetRead 47 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 48 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 49 /* jump0 */ -#define OP_FkIfZero 50 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IFindKey 47 /* jump */ +#define OP_RowSetRead 48 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 49 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 50 /* jump0 */ #define OP_IsNull 51 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 52 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 53 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -17488,49 +17722,49 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 57 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ #define OP_ElseEq 59 /* jump, same as TK_ESCAPE */ -#define OP_IfPos 60 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ -#define OP_IfNotZero 61 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 62 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 63 /* jump */ -#define OP_VNext 64 /* jump */ -#define OP_Filter 65 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ -#define OP_PureFunc 66 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Function 67 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Return 68 -#define OP_EndCoroutine 69 -#define OP_HaltIfNull 70 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 71 -#define OP_Integer 72 /* synopsis: r[P2]=P1 */ -#define OP_Int64 73 /* synopsis: r[P2]=P4 */ -#define OP_String 74 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_BeginSubrtn 75 /* synopsis: r[P2]=NULL */ -#define OP_Null 76 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 77 /* synopsis: r[P1]=NULL */ -#define OP_Blob 78 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 79 /* synopsis: r[P2]=parameter(P1) */ -#define OP_Move 80 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 81 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 82 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 83 /* synopsis: r[P2]=r[P1] */ -#define OP_FkCheck 84 -#define OP_ResultRow 85 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 86 -#define OP_AddImm 87 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 88 -#define OP_Cast 89 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 90 -#define OP_Compare 91 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_IsTrue 92 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_ZeroOrNull 93 /* synopsis: r[P2] = 0 OR NULL */ -#define OP_Offset 94 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 95 /* synopsis: r[P3]=PX cursor P1 column P2 */ -#define OP_TypeCheck 96 /* synopsis: typecheck(r[P1@P2]) */ -#define OP_Affinity 97 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 98 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 99 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 100 -#define OP_SetCookie 101 -#define OP_ReopenIdx 102 /* synopsis: root=P2 iDb=P3 */ +#define OP_FkIfZero 60 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 61 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 62 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 63 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 64 /* jump */ +#define OP_VNext 65 /* jump */ +#define OP_Filter 66 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ +#define OP_PureFunc 67 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 68 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 69 +#define OP_EndCoroutine 70 +#define OP_HaltIfNull 71 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 72 +#define OP_Integer 73 /* synopsis: r[P2]=P1 */ +#define OP_Int64 74 /* synopsis: r[P2]=P4 */ +#define OP_String 75 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_BeginSubrtn 76 /* synopsis: r[P2]=NULL */ +#define OP_Null 77 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 78 /* synopsis: r[P1]=NULL */ +#define OP_Blob 79 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 80 /* synopsis: r[P2]=parameter(P1) */ +#define OP_Move 81 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 82 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 83 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 84 /* synopsis: r[P2]=r[P1] */ +#define OP_FkCheck 85 +#define OP_ResultRow 86 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 87 +#define OP_AddImm 88 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 89 +#define OP_Cast 90 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 91 +#define OP_Compare 92 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 93 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 94 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 95 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 96 /* synopsis: r[P3]=PX cursor P1 column P2 */ +#define OP_TypeCheck 97 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 98 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 99 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 100 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 101 +#define OP_SetCookie 102 #define OP_BitAnd 103 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 104 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 105 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 162 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 164 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 165 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 166 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 167 -#define OP_CursorLock 168 -#define OP_CursorUnlock 169 -#define OP_TableLock 170 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 171 -#define OP_VCreate 172 -#define OP_VDestroy 173 -#define OP_VOpen 174 -#define OP_VCheck 175 -#define OP_VInitIn 176 /* synopsis: r[P2]=ValueList(P1,P3) */ -#define OP_VColumn 177 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 178 -#define OP_Pagecount 179 -#define OP_MaxPgcnt 180 -#define OP_ClrSubtype 181 /* synopsis: r[P1].subtype = 0 */ -#define OP_GetSubtype 182 /* synopsis: r[P2] = r[P1].subtype */ -#define OP_SetSubtype 183 /* synopsis: r[P2].subtype = r[P1] */ -#define OP_FilterAdd 184 /* synopsis: filter(P1) += key(P3@P4) */ -#define OP_Trace 185 -#define OP_CursorHint 186 -#define OP_ReleaseReg 187 /* synopsis: release r[P1@P2] mask P3 */ -#define OP_Noop 188 -#define OP_Explain 189 -#define OP_Abortable 190 +#define OP_DropIndex 155 +#define OP_DropTrigger 156 +#define OP_IntegrityCk 157 +#define OP_RowSetAdd 158 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 159 +#define OP_FkCounter 160 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 161 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 162 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 163 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 164 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 165 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 166 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 167 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 168 +#define OP_CursorLock 169 +#define OP_CursorUnlock 170 +#define OP_TableLock 171 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 172 +#define OP_VCreate 173 +#define OP_VDestroy 174 +#define OP_VOpen 175 +#define OP_VCheck 176 +#define OP_VInitIn 177 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 178 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 179 +#define OP_Pagecount 180 +#define OP_MaxPgcnt 181 +#define OP_ClrSubtype 182 /* synopsis: r[P1].subtype = 0 */ +#define OP_GetSubtype 183 /* synopsis: r[P2] = r[P1].subtype */ +#define OP_SetSubtype 184 /* synopsis: r[P2].subtype = r[P1] */ +#define OP_FilterAdd 185 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 186 +#define OP_CursorHint 187 +#define OP_ReleaseReg 188 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 189 +#define OP_Explain 190 +#define OP_Abortable 191 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -17638,25 +17873,26 @@ typedef struct VdbeOpList VdbeOpList; /* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ /* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ /* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x01, 0x41,\ -/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x41, 0x23,\ -/* 48 */ 0x0b, 0x81, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01,\ -/* 64 */ 0x41, 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00,\ -/* 72 */ 0x10, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10,\ -/* 80 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02,\ -/* 88 */ 0x02, 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40,\ -/* 96 */ 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x26,\ +/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x41, 0x09,\ +/* 48 */ 0x23, 0x0b, 0x81, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x01, 0x03, 0x03, 0x03,\ +/* 64 */ 0x01, 0x41, 0x01, 0x00, 0x00, 0x02, 0x02, 0x08,\ +/* 72 */ 0x00, 0x10, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10,\ +/* 80 */ 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00,\ +/* 88 */ 0x02, 0x02, 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20,\ +/* 96 */ 0x40, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x26,\ /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 112 */ 0x26, 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40,\ -/* 120 */ 0x00, 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10,\ -/* 128 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00,\ -/* 136 */ 0x50, 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50,\ -/* 144 */ 0x40, 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00,\ -/* 160 */ 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10,\ -/* 176 */ 0x50, 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12,\ -/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} +/* 112 */ 0x26, 0x40, 0x40, 0x12, 0x00, 0x40, 0x10, 0x40,\ +/* 120 */ 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x40, 0x40,\ +/* 128 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,\ +/* 136 */ 0x00, 0x50, 0x00, 0x40, 0x04, 0x04, 0x00, 0x40,\ +/* 144 */ 0x50, 0x40, 0x10, 0x00, 0x00, 0x10, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x06, 0x10,\ +/* 160 */ 0x00, 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,\ +/* 176 */ 0x10, 0x50, 0x40, 0x00, 0x10, 0x10, 0x02, 0x12,\ +/* 184 */ 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +} /* The resolve3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -17664,7 +17900,7 @@ typedef struct VdbeOpList VdbeOpList; ** generated this include file strives to group all JUMP opcodes ** together near the beginning of the list. */ -#define SQLITE_MX_JUMP_OPCODE 65 /* Maximum JUMP opcode */ +#define SQLITE_MX_JUMP_OPCODE 66 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -21589,17 +21825,22 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; +/* +** Maxium number of base-10 digits in an unsigned 64-bit integer +*/ +#define SQLITE_U64_DIGITS 20 + /* ** An instance of this object receives the decoding of a floating point ** value into an approximate decimal representation. */ struct FpDecode { - int n; /* Significant digits in the decode */ - int iDP; /* Location of the decimal point */ - char *z; /* Start of significant digits */ - char zBuf[20]; /* Storage for significant digits */ - char sign; /* '+' or '-' */ - char isSpecial; /* 1: Infinity 2: NaN */ + int n; /* Significant digits in the decode */ + int iDP; /* Location of the decimal point */ + char *z; /* Start of significant digits */ + char zBuf[SQLITE_U64_DIGITS+1]; /* Storage for significant digits */ + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ }; SQLITE_PRIVATE void sqlite3FpDecode(FpDecode*,double,int,int); @@ -22287,6 +22528,7 @@ SQLITE_PRIVATE char *sqlite3RCStrResize(char*,u64); SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); +SQLITE_PRIVATE int sqlite3StrAccumEnlargeIfNeeded(StrAccum*, i64); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); @@ -23514,6 +23756,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_STMTJRNL_SPILL "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), #endif +#ifdef SQLITE_STRICT_SUBTYPE + "STRICT_SUBTYPE", +#endif #ifdef SQLITE_SUBSTR_COMPATIBILITY "SUBSTR_COMPATIBILITY", #endif @@ -24655,7 +24900,7 @@ SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem*); SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); -SQLITE_PRIVATE SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem*, int*); +SQLITE_PRIVATE int sqlite3MemRealValueRC(Mem*, double*); SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); @@ -24686,6 +24931,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int,int); #endif SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); +SQLITE_PRIVATE int sqlite3VdbeFindIndexKey(BtCursor*, Index*, UnpackedRecord*, int*, int); SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); @@ -32269,7 +32515,7 @@ static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ sqlite3StrAccumSetError(pAccum, SQLITE_TOOBIG); return 0; } - z = sqlite3DbMallocRaw(pAccum->db, n); + z = sqlite3_malloc(n); if( z==0 ){ sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); } @@ -32600,9 +32846,11 @@ SQLITE_API void sqlite3_str_vappendf( }while( longvalue>0 ); } length = (int)(&zOut[nOut-1]-bufpt); - while( precision>length ){ - *(--bufpt) = '0'; /* Zero pad */ - length++; + if( precision>length ){ /* zero pad */ + int nn = precision-length; + bufpt -= nn; + memset(bufpt,'0',nn); + length = precision; } if( cThousand ){ int nn = (length - 1)/3; /* Number of "," to insert */ @@ -32633,6 +32881,7 @@ SQLITE_API void sqlite3_str_vappendf( FpDecode s; int iRound; int j; + i64 szBufNeeded; /* Size needed to hold the output */ if( bArgList ){ realvalue = getDoubleArg(pArgList); @@ -32721,17 +32970,31 @@ SQLITE_API void sqlite3_str_vappendf( }else{ e2 = s.iDP - 1; } - bufpt = buf; - { - i64 szBufNeeded; /* Size of a temporary buffer needed */ - szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; - if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; - if( szBufNeeded > etBUFSIZE ){ - bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); - if( bufpt==0 ) return; + + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+10; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; + if( szBufNeeded + pAccum->nChar >= pAccum->nAlloc ){ + if( pAccum->mxAlloc==0 && pAccum->accError==0 ){ + /* Unable to allocate space in pAccum, perhaps because it + ** is coming from sqlite3_snprintf() or similar. We'll have + ** to render into temporary space and the memcpy() it over. */ + bufpt = sqlite3_malloc(szBufNeeded); + if( bufpt==0 ){ + sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); + return; + } + zExtra = bufpt; + }else if( sqlite3StrAccumEnlarge(pAccum, szBufNeeded)zText + pAccum->nChar; } + }else{ + bufpt = pAccum->zText + pAccum->nChar; } zOut = bufpt; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ @@ -32739,12 +33002,24 @@ SQLITE_API void sqlite3_str_vappendf( } /* Digits prior to the decimal point */ j = 0; + assert( s.n>0 ); if( e2<0 ){ *(bufpt++) = '0'; - }else{ + }else if( cThousand ){ for(; e2>=0; e2--){ *(bufpt++) = j1 ) *(bufpt++) = ','; + if( (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + } + }else{ + j = e2+1; + if( j>s.n ) j = s.n; + memcpy(bufpt, s.z, j); + bufpt += j; + e2 -= j; + if( e2>=0 ){ + memset(bufpt, '0', e2+1); + bufpt += e2+1; + e2 = -1; } } /* The decimal point */ @@ -32753,12 +33028,26 @@ SQLITE_API void sqlite3_str_vappendf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0 && precision>0; precision--, e2++){ - *(bufpt++) = '0'; + if( e2<(-1) && precision>0 ){ + int nn = -1-e2; + if( nn>precision ) nn = precision; + memset(bufpt, '0', nn); + bufpt += nn; + precision -= nn; } /* Significant digits after the decimal point */ - while( (precision--)>0 ){ - *(bufpt++) = j0 ){ + int nn = s.n - j; + if( NEVER(nn>precision) ) nn = precision; + if( nn>0 ){ + memcpy(bufpt, s.z+j, nn); + bufpt += nn; + precision -= nn; + } + if( precision>0 && !flag_rtz ){ + memset(bufpt, '0', precision); + bufpt += precision; + } } /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ @@ -32788,27 +33077,39 @@ SQLITE_API void sqlite3_str_vappendf( *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ } - *bufpt = 0; - /* The converted number is in buf[] and zero terminated. Output it. - ** Note that the number is in the usual order, not reversed as with - ** integer conversions. */ length = (int)(bufpt-zOut); - bufpt = zOut; - - /* Special case: Add leading zeros if the flag_zeropad flag is - ** set and we are not left justified */ - if( flag_zeropad && !flag_leftjustify && length < width){ - int i; - int nPad = width - length; - for(i=width; i>=nPad; i--){ - bufpt[i] = bufpt[i-nPad]; + assert( length <= szBufNeeded ); + if( lengthnChar += length; + zOut[length] = 0; + continue; + }else{ + /* We were unable to render directly into pAccum because we + ** couldn't allocate sufficient memory. We need to memcpy() + ** the rendering (or some prefix thereof) into the output + ** buffer. */ + bufpt[0] = 0; + bufpt = zExtra; + break; + } } case etSIZE: if( !bArgList ){ @@ -32852,10 +33153,9 @@ SQLITE_API void sqlite3_str_vappendf( i64 nCopyBytes; if( nPrior > precision-1 ) nPrior = precision - 1; nCopyBytes = length*nPrior; - if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ - sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + if( sqlite3StrAccumEnlargeIfNeeded(pAccum, nCopyBytes) ){ + break; } - if( pAccum->accError ) break; sqlite3_str_append(pAccum, &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); precision -= nPrior; @@ -33202,6 +33502,13 @@ SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ return (int)N; } +SQLITE_PRIVATE int sqlite3StrAccumEnlargeIfNeeded(StrAccum *p, i64 N){ + if( N + p->nChar >= p->nAlloc ){ + sqlite3StrAccumEnlarge(p, N); + } + return p->accError; +} + /* ** Append N copies of character c to the given string buffer. */ @@ -33365,7 +33672,7 @@ SQLITE_API void sqlite3_str_reset(StrAccum *p){ ** of its content, all in one call. */ SQLITE_API void sqlite3_str_free(sqlite3_str *p){ - if( p ){ + if( p!=0 && p!=&sqlite3OomStr ){ sqlite3_str_reset(p); sqlite3_free(p); } @@ -35371,6 +35678,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); bRc = CloseHandle((HANDLE)p->tid); + (void)bRc; /* Prevent warning when assert() is a no-op */ assert( bRc ); } if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; @@ -36510,29 +36818,92 @@ SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ return h; } +#if !defined(SQLITE_DISABLE_INTRINSIC) \ + && (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen>32))) +#define SQLITE_USE_UINT128 +#endif + /* -** Two inputs are multiplied to get a 128-bit result. Return -** the high-order 64 bits of that result. +** Two inputs are multiplied to get a 128-bit result. Write the +** lower 64-bits of the result into *pLo, and return the high-order +** 64 bits. */ -static u64 sqlite3Multiply128(u64 a, u64 b){ -#if (defined(__GNUC__) || defined(__clang__)) \ - && (defined(__x86_64__) || defined(__aarch64__) || defined(__riscv)) - return ((__uint128_t)a * b) >> 64; -#elif defined(_MSC_VER) && defined(_M_X64) +static u64 sqlite3Multiply128(u64 a, u64 b, u64 *pLo){ +#if defined(SQLITE_USE_UINT128) + __uint128_t r = (__uint128_t)a * b; + *pLo = (u64)r; + return (u64)(r>>64); +#elif defined(_WIN64) && !defined(SQLITE_DISABLE_INTRINSIC) + *pLo = a*b; return __umulh(a, b); #else - u64 a1 = (u32)a; - u64 a2 = a >> 32; - u64 b1 = (u32)b; - u64 b2 = b >> 32; - u64 p0 = a1 * b1; - u64 p1 = a1 * b2; - u64 p2 = a2 * b1; - u64 p3 = a2 * b2; - u64 carry = ((p0 >> 32) + (u32)p1 + (u32)p2) >> 32; - return p3 + (p1 >> 32) + (p2 >> 32) + carry; -#endif -} + u64 a0 = (u32)a; + u64 a1 = a >> 32; + u64 b0 = (u32)b; + u64 b1 = b >> 32; + u64 a0b0 = a0 * b0; + u64 a1b1 = a1 * b1; + u64 a0b1 = a0 * b1; + u64 a1b0 = a1 * b0; + u64 t = (a0b0 >> 32) + (u32)a0b1 + (u32)a1b0; + *pLo = (a0b0 & UINT64_C(0xffffffff)) | (t << 32); + return a1b1 + (a0b1>>32) + (a1b0>>32) + (t>>32); +#endif +} + +/* +** A is an unsigned 96-bit integer formed by (a<<32)+aLo. +** B is an unsigned 64-bit integer. +** +** Compute the upper 96 bits of 160-bit result of A*B. +** +** Write ((A*B)>>64 & 0xffffffff) (the middle 32 bits of A*B) +** into *pLo. Return the upper 64 bits of A*B. +** +** The lower 64 bits of A*B are discarded. +*/ +static u64 sqlite3Multiply160(u64 a, u32 aLo, u64 b, u32 *pLo){ +#if defined(SQLITE_USE_UINT128) + __uint128_t r = (__uint128_t)a * b; + r += ((__uint128_t)aLo * b) >> 32; + *pLo = (r>>32)&0xffffffff; + return r>>64; +#elif defined(_WIN64) && !defined(SQLITE_DISABLE_INTRINSIC) + u64 r1_hi = __umulh(a,b); + u64 r1_lo = a*b; + u64 r2 = (__umulh((u64)aLo,b)<<32) + ((aLo*b)>>32); + u64 t = r1_lo + r2; + if( t>32; + return r1_hi; +#else + u64 x2 = a>>32; + u64 x1 = a&0xffffffff; + u64 x0 = aLo; + u64 y1 = b>>32; + u64 y0 = b&0xffffffff; + u64 x2y1 = x2*y1; + u64 r4 = x2y1>>32; + u64 x2y0 = x2*y0; + u64 x1y1 = x1*y1; + u64 r3 = (x2y1 & 0xffffffff) + (x2y0 >>32) + (x1y1 >>32); + u64 x1y0 = x1*y0; + u64 x0y1 = x0*y1; + u64 r2 = (x2y0 & 0xffffffff) + (x1y1 & 0xffffffff) + + (x1y0 >>32) + (x0y1>>32); + u64 x0y0 = x0*y0; + u64 r1 = (x1y0 & 0xffffffff) + (x0y1 & 0xffffffff) + + (x0y0 >>32); + r2 += r1>>32; + r3 += r2>>32; + *pLo = r2&0xffffffff; + return (r4<<32) + r3; +#endif +} + +#undef SQLITE_USE_UINT128 /* ** Return a u64 with the N-th bit set. @@ -36556,6 +36927,9 @@ static u64 sqlite3Multiply128(u64 a, u64 b){ ** as appropriate so the most significant 64 bits fit exactly into a ** 64-bit unsigned integer. ** +** Write into *pLo the next 32 significant bits of the answer after +** the first 64. +** ** Algorithm: ** ** (1) For p between 0 and 26, return the value directly from the aBase[] @@ -36564,70 +36938,106 @@ static u64 sqlite3Multiply128(u64 a, u64 b){ ** (2) For p outside the range 0 to 26, use aScale[] for the initial value ** then refine that result (if necessary) by a single multiplication ** against aBase[]. +** +** The constant tables aBase[], aScale[], and aScaleLo[] are generated +** by the C program at ../tool/mkfptab.c run with the --round option. */ -static u64 powerOfTen(int p){ +static u64 powerOfTen(int p, u32 *pLo){ static const u64 aBase[] = { - 0x8000000000000000LLU, /* 0: 1.0e+0 << 63 */ - 0xa000000000000000LLU, /* 1: 1.0e+1 << 60 */ - 0xc800000000000000LLU, /* 2: 1.0e+2 << 57 */ - 0xfa00000000000000LLU, /* 3: 1.0e+3 << 54 */ - 0x9c40000000000000LLU, /* 4: 1.0e+4 << 50 */ - 0xc350000000000000LLU, /* 5: 1.0e+5 << 47 */ - 0xf424000000000000LLU, /* 6: 1.0e+6 << 44 */ - 0x9896800000000000LLU, /* 7: 1.0e+7 << 40 */ - 0xbebc200000000000LLU, /* 8: 1.0e+8 << 37 */ - 0xee6b280000000000LLU, /* 9: 1.0e+9 << 34 */ - 0x9502f90000000000LLU, /* 10: 1.0e+10 << 30 */ - 0xba43b74000000000LLU, /* 11: 1.0e+11 << 27 */ - 0xe8d4a51000000000LLU, /* 12: 1.0e+12 << 24 */ - 0x9184e72a00000000LLU, /* 13: 1.0e+13 << 20 */ - 0xb5e620f480000000LLU, /* 14: 1.0e+14 << 17 */ - 0xe35fa931a0000000LLU, /* 15: 1.0e+15 << 14 */ - 0x8e1bc9bf04000000LLU, /* 16: 1.0e+16 << 10 */ - 0xb1a2bc2ec5000000LLU, /* 17: 1.0e+17 << 7 */ - 0xde0b6b3a76400000LLU, /* 18: 1.0e+18 << 4 */ - 0x8ac7230489e80000LLU, /* 19: 1.0e+19 >> 0 */ - 0xad78ebc5ac620000LLU, /* 20: 1.0e+20 >> 3 */ - 0xd8d726b7177a8000LLU, /* 21: 1.0e+21 >> 6 */ - 0x878678326eac9000LLU, /* 22: 1.0e+22 >> 10 */ - 0xa968163f0a57b400LLU, /* 23: 1.0e+23 >> 13 */ - 0xd3c21bcecceda100LLU, /* 24: 1.0e+24 >> 16 */ - 0x84595161401484a0LLU, /* 25: 1.0e+25 >> 20 */ - 0xa56fa5b99019a5c8LLU, /* 26: 1.0e+26 >> 23 */ + UINT64_C(0x8000000000000000), /* 0: 1.0e+0 << 63 */ + UINT64_C(0xa000000000000000), /* 1: 1.0e+1 << 60 */ + UINT64_C(0xc800000000000000), /* 2: 1.0e+2 << 57 */ + UINT64_C(0xfa00000000000000), /* 3: 1.0e+3 << 54 */ + UINT64_C(0x9c40000000000000), /* 4: 1.0e+4 << 50 */ + UINT64_C(0xc350000000000000), /* 5: 1.0e+5 << 47 */ + UINT64_C(0xf424000000000000), /* 6: 1.0e+6 << 44 */ + UINT64_C(0x9896800000000000), /* 7: 1.0e+7 << 40 */ + UINT64_C(0xbebc200000000000), /* 8: 1.0e+8 << 37 */ + UINT64_C(0xee6b280000000000), /* 9: 1.0e+9 << 34 */ + UINT64_C(0x9502f90000000000), /* 10: 1.0e+10 << 30 */ + UINT64_C(0xba43b74000000000), /* 11: 1.0e+11 << 27 */ + UINT64_C(0xe8d4a51000000000), /* 12: 1.0e+12 << 24 */ + UINT64_C(0x9184e72a00000000), /* 13: 1.0e+13 << 20 */ + UINT64_C(0xb5e620f480000000), /* 14: 1.0e+14 << 17 */ + UINT64_C(0xe35fa931a0000000), /* 15: 1.0e+15 << 14 */ + UINT64_C(0x8e1bc9bf04000000), /* 16: 1.0e+16 << 10 */ + UINT64_C(0xb1a2bc2ec5000000), /* 17: 1.0e+17 << 7 */ + UINT64_C(0xde0b6b3a76400000), /* 18: 1.0e+18 << 4 */ + UINT64_C(0x8ac7230489e80000), /* 19: 1.0e+19 >> 0 */ + UINT64_C(0xad78ebc5ac620000), /* 20: 1.0e+20 >> 3 */ + UINT64_C(0xd8d726b7177a8000), /* 21: 1.0e+21 >> 6 */ + UINT64_C(0x878678326eac9000), /* 22: 1.0e+22 >> 10 */ + UINT64_C(0xa968163f0a57b400), /* 23: 1.0e+23 >> 13 */ + UINT64_C(0xd3c21bcecceda100), /* 24: 1.0e+24 >> 16 */ + UINT64_C(0x84595161401484a0), /* 25: 1.0e+25 >> 20 */ + UINT64_C(0xa56fa5b99019a5c8), /* 26: 1.0e+26 >> 23 */ }; static const u64 aScale[] = { - 0x8049a4ac0c5811aeLLU, /* 0: 1.0e-351 << 1229 */ - 0xcf42894a5dce35eaLLU, /* 1: 1.0e-324 << 1140 */ - 0xa76c582338ed2622LLU, /* 2: 1.0e-297 << 1050 */ - 0x873e4f75e2224e68LLU, /* 3: 1.0e-270 << 960 */ - 0xda7f5bf590966849LLU, /* 4: 1.0e-243 << 871 */ - 0xb080392cc4349dedLLU, /* 5: 1.0e-216 << 781 */ - 0x8e938662882af53eLLU, /* 6: 1.0e-189 << 691 */ - 0xe65829b3046b0afaLLU, /* 7: 1.0e-162 << 602 */ - 0xba121a4650e4ddecLLU, /* 8: 1.0e-135 << 512 */ - 0x964e858c91ba2655LLU, /* 9: 1.0e-108 << 422 */ - 0xf2d56790ab41c2a3LLU, /* 10: 1.0e-81 << 333 */ - 0xc428d05aa4751e4dLLU, /* 11: 1.0e-54 << 243 */ - 0x9e74d1b791e07e48LLU, /* 12: 1.0e-27 << 153 */ - 0x8000000000000000LLU, /* 13: 1.0e+0 << 63 */ - 0xcecb8f27f4200f3aLLU, /* 14: 1.0e+27 >> 26 */ - 0xa70c3c40a64e6c52LLU, /* 15: 1.0e+54 >> 116 */ - 0x86f0ac99b4e8dafdLLU, /* 16: 1.0e+81 >> 206 */ - 0xda01ee641a708deaLLU, /* 17: 1.0e+108 >> 295 */ - 0xb01ae745b101e9e4LLU, /* 18: 1.0e+135 >> 385 */ - 0x8e41ade9fbebc27dLLU, /* 19: 1.0e+162 >> 475 */ - 0xe5d3ef282a242e82LLU, /* 20: 1.0e+189 >> 564 */ - 0xb9a74a0637ce2ee1LLU, /* 21: 1.0e+216 >> 654 */ - 0x95f83d0a1fb69cd9LLU, /* 22: 1.0e+243 >> 744 */ - 0xf24a01a73cf2dcd0LLU, /* 23: 1.0e+270 >> 833 */ - 0xc3b8358109e84f07LLU, /* 24: 1.0e+297 >> 923 */ - 0x9e19db92b4e31ba9LLU, /* 25: 1.0e+324 >> 1013 */ + UINT64_C(0x8049a4ac0c5811ae), /* 0: 1.0e-351 << 1229 */ + UINT64_C(0xcf42894a5dce35ea), /* 1: 1.0e-324 << 1140 */ + UINT64_C(0xa76c582338ed2621), /* 2: 1.0e-297 << 1050 */ + UINT64_C(0x873e4f75e2224e68), /* 3: 1.0e-270 << 960 */ + UINT64_C(0xda7f5bf590966848), /* 4: 1.0e-243 << 871 */ + UINT64_C(0xb080392cc4349dec), /* 5: 1.0e-216 << 781 */ + UINT64_C(0x8e938662882af53e), /* 6: 1.0e-189 << 691 */ + UINT64_C(0xe65829b3046b0afa), /* 7: 1.0e-162 << 602 */ + UINT64_C(0xba121a4650e4ddeb), /* 8: 1.0e-135 << 512 */ + UINT64_C(0x964e858c91ba2655), /* 9: 1.0e-108 << 422 */ + UINT64_C(0xf2d56790ab41c2a2), /* 10: 1.0e-81 << 333 */ + UINT64_C(0xc428d05aa4751e4c), /* 11: 1.0e-54 << 243 */ + UINT64_C(0x9e74d1b791e07e48), /* 12: 1.0e-27 << 153 */ + UINT64_C(0xcccccccccccccccc), /* 13: 1.0e-1 << 67 (special case) */ + UINT64_C(0xcecb8f27f4200f3a), /* 14: 1.0e+27 >> 26 */ + UINT64_C(0xa70c3c40a64e6c51), /* 15: 1.0e+54 >> 116 */ + UINT64_C(0x86f0ac99b4e8dafd), /* 16: 1.0e+81 >> 206 */ + UINT64_C(0xda01ee641a708de9), /* 17: 1.0e+108 >> 295 */ + UINT64_C(0xb01ae745b101e9e4), /* 18: 1.0e+135 >> 385 */ + UINT64_C(0x8e41ade9fbebc27d), /* 19: 1.0e+162 >> 475 */ + UINT64_C(0xe5d3ef282a242e81), /* 20: 1.0e+189 >> 564 */ + UINT64_C(0xb9a74a0637ce2ee1), /* 21: 1.0e+216 >> 654 */ + UINT64_C(0x95f83d0a1fb69cd9), /* 22: 1.0e+243 >> 744 */ + UINT64_C(0xf24a01a73cf2dccf), /* 23: 1.0e+270 >> 833 */ + UINT64_C(0xc3b8358109e84f07), /* 24: 1.0e+297 >> 923 */ + UINT64_C(0x9e19db92b4e31ba9), /* 25: 1.0e+324 >> 1013 */ + }; + static const unsigned int aScaleLo[] = { + 0x205b896d, /* 0: 1.0e-351 << 1229 */ + 0x52064cad, /* 1: 1.0e-324 << 1140 */ + 0xaf2af2b8, /* 2: 1.0e-297 << 1050 */ + 0x5a7744a7, /* 3: 1.0e-270 << 960 */ + 0xaf39a475, /* 4: 1.0e-243 << 871 */ + 0xbd8d794e, /* 5: 1.0e-216 << 781 */ + 0x547eb47b, /* 6: 1.0e-189 << 691 */ + 0x0cb4a5a3, /* 7: 1.0e-162 << 602 */ + 0x92f34d62, /* 8: 1.0e-135 << 512 */ + 0x3a6a07f9, /* 9: 1.0e-108 << 422 */ + 0xfae27299, /* 10: 1.0e-81 << 333 */ + 0xaa97e14c, /* 11: 1.0e-54 << 243 */ + 0x775ea265, /* 12: 1.0e-27 << 153 */ + 0xcccccccc, /* 13: 1.0e-1 << 67 (special case) */ + 0x00000000, /* 14: 1.0e+27 >> 26 */ + 0x999090b6, /* 15: 1.0e+54 >> 116 */ + 0x69a028bb, /* 16: 1.0e+81 >> 206 */ + 0xe80e6f48, /* 17: 1.0e+108 >> 295 */ + 0x5ec05dd0, /* 18: 1.0e+135 >> 385 */ + 0x14588f14, /* 19: 1.0e+162 >> 475 */ + 0x8f1668c9, /* 20: 1.0e+189 >> 564 */ + 0x6d953e2c, /* 21: 1.0e+216 >> 654 */ + 0x4abdaf10, /* 22: 1.0e+243 >> 744 */ + 0xbc633b39, /* 23: 1.0e+270 >> 833 */ + 0x0a862f81, /* 24: 1.0e+297 >> 923 */ + 0x6c07a2c2, /* 25: 1.0e+324 >> 1013 */ }; int g, n; - u64 x, y; + u64 s, x; + u32 lo; assert( p>=POWERSOF10_FIRST && p<=POWERSOF10_LAST ); if( p<0 ){ + if( p==(-1) ){ + *pLo = aScaleLo[13]; + return aScale[13]; + } g = p/27; n = p%27; if( n ){ @@ -36635,19 +37045,23 @@ static u64 powerOfTen(int p){ n += 27; } }else if( p<27 ){ + *pLo = 0; return aBase[p]; }else{ g = p/27; n = p%27; } - y = aScale[g+13]; + s = aScale[g+13]; if( n==0 ){ - return y; + *pLo = aScaleLo[g+13]; + return s; } - x = sqlite3Multiply128(aBase[n],y); + x = sqlite3Multiply160(s,aScaleLo[g+13],aBase[n],&lo); if( (U64_BIT(63) & x)==0 ){ - x = (x<<1)|1; + x = x<<1 | ((lo>>31)&1); + lo = (lo<<1) | 1; } + *pLo = lo; return x; } @@ -36676,7 +37090,8 @@ static int pwr2to10(int p){ return (p*78913) >> 18; } ** Count leading zeros for a 64-bit unsigned integer. */ static int countLeadingZeros(u64 m){ -#if defined(__GNUC__) || defined(__clang__) +#if (defined(__GNUC__) || defined(__clang__)) \ + && !defined(SQLITE_DISABLE_INTRINSIC) return __builtin_clzll(m); #else int n = 0; @@ -36700,10 +37115,11 @@ static int countLeadingZeros(u64 m){ */ static void sqlite3Fp2Convert10(u64 m, int e, int n, u64 *pD, int *pP){ int p; - u64 h; + u64 h, d1; + u32 d2; assert( n>=1 && n<=18 ); p = n - 1 - pwr2to10(e+63); - h = sqlite3Multiply128(m, powerOfTen(p)); + h = sqlite3Multiply128(m, powerOfTen(p,&d2), &d1); assert( -(e + pwr10to2(p) + 2) >= 0 ); assert( -(e + pwr10to2(p) + 1) <= 63 ); if( n==18 ){ @@ -36717,47 +37133,51 @@ static void sqlite3Fp2Convert10(u64 m, int e, int n, u64 *pD, int *pP){ /* ** Return an IEEE754 floating point value that approximates d*pow(10,p). +** +** The (current) algorithm is adapted from the work of Ross Cox at +** https://github.com/rsc/fpfmt */ static double sqlite3Fp10Convert2(u64 d, int p){ - u64 out; - int e1; - int lz; - int lp; - int x; - u64 h; + int b, lp, e, adj, s; + u32 pwr10l, mid1; + u64 pwr10h, x, hi, lo, sticky, u, m; double r; - assert( (d & U64_BIT(63))==0 ); - assert( d!=0 ); - if( pPOWERSOF10_LAST ){ - return INFINITY; - } - lz = countLeadingZeros(d); + if( pPOWERSOF10_LAST ) return INFINITY; + b = 64 - countLeadingZeros(d); lp = pwr10to2(p); - e1 = lz - (lp + 11); - if( e1>1074 ){ - if( e1>=1130 ) return 0.0; - e1 = 1074; - } - h = sqlite3Multiply128(d<= 0 ); - assert( x <= 63 ); - out = h >> x; - if( out >= U64_BIT(55)-2 ){ - out >>= 1; - e1--; - } - if( e1<=(-972) ){ - return INFINITY; - } - out = (out + 2) >> 2; - if( (out & U64_BIT(52))!=0 ){ - out = (out & ~U64_BIT(52)) | ((u64)(1075-e1)<<52); - } - memcpy(&r, &out, 8); + e = 53 - b - lp; + if( e > 1074 ){ + if( e>=1130 ) return 0.0; + e = 1074; + } + s = -(e-(64-b) + lp + 3); + pwr10h = powerOfTen(p, &pwr10l); + if( pwr10l!=0 ){ + pwr10h++; + pwr10l = ~pwr10l; + } + x = d<<(64-b); + hi = sqlite3Multiply128(x,pwr10h,&lo); + mid1 = lo>>32; + sticky = 1; + if( (hi & (U64_BIT(s)-1))==0 ) { + u32 mid2 = sqlite3Multiply128(x,((u64)pwr10l)<<32,&lo)>>32; + sticky = (mid1-mid2 > 1); + hi -= mid1 < mid2; + } + u = (hi>>s) | sticky; + adj = (u >= U64_BIT(55)-2); + if( adj ){ + u = (u>>adj) | (u&1); + e -= adj; + } + m = (u + 1 + ((u>>2)&1)) >> 2; + if( e<=(-972) ) return INFINITY; + if((m & U64_BIT(52)) != 0){ + m = (m & ~U64_BIT(52)) | ((u64)(1075-e)<<52); + } + memcpy(&r,&m,8); return r; } @@ -36767,134 +37187,164 @@ static double sqlite3Fp10Convert2(u64 d, int p){ ** ** z[] must be UTF-8 and zero-terminated. ** -** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. More specifically -** return -** 1 => The input string is a pure integer -** 2 or more => The input has a decimal point or eNNN clause -** 0 or less => The input string is not a valid number -** -1 => Not a valid number, but has a valid prefix which -** includes a decimal point and/or an eNNN clause -** -** Valid numbers are in one of these formats: +** Return positive if the result is a valid real number (or integer) and +** zero or negative if the string is empty or contains extraneous text. +** Lower bits of the return value contain addition information about the +** parse: +** +** bit 0 => Set if any prefix of the input is valid. Clear if +** there is no prefix of the input that can be seen as +** a valid floating point number. +** bit 1 => Set if the input contains a decimal point or eNNN +** clause. Zero if the input is an integer. +** bit 2 => The input is exactly 0.0, not an underflow from +** some value near zero. +** bit 3 => Set if there are more than about 19 significant +** digits in the input. +** +** If the input contains a syntax error but begins with text that might +** be a valid number of some kind, then the result is negative. The +** result is only zero if no prefix of the input could be interpreted as +** a number. +** +** Leading and trailing whitespace is ignored. Valid numbers are in +** one of the formats below: ** ** [+-]digits[E[+-]digits] ** [+-]digits.[digits][E[+-]digits] ** [+-].digits[E[+-]digits] ** -** Leading and trailing whitespace is ignored for the purpose of determining -** validity. -** -** If some prefix of the input string is a valid number, this routine -** returns FALSE but it still converts the prefix and writes the result -** into *pResult. -*/ -#if defined(_MSC_VER) -#pragma warning(disable : 4756) -#endif -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){ +** Algorithm sketch: Compute an unsigned 64-bit integer s and a base-10 +** exponent d such that the value encoding by the input is s*pow(10,d). +** Then invoke sqlite3Fp10Convert2() to calculated the closest possible +** IEEE754 double. The sign is added back afterwards, if the input string +** starts with a "-". The use of an unsigned 64-bit s mantissa means that +** only about the first 19 significant digits of the input can contribute +** to the result. This can result in suboptimal rounding decisions when +** correct rounding requires more than 19 input digits. For example, +** this routine renders "3500000000000000.2500001" as +** 3500000000000000.0 instead of 3500000000000000.5 because the decision +** to round up instead of using banker's rounding to round down is determined +** by the 23rd significant digit, which this routine ignores. It is not +** possible to do better without some kind of BigNum. +*/ +SQLITE_PRIVATE int sqlite3AtoF(const char *zIn, double *pResult){ #ifndef SQLITE_OMIT_FLOATING_POINT - /* sign * significand * (10 ^ (esign * exponent)) */ - int neg = 0; /* True for a negative value */ - u64 s = 0; /* mantissa */ - int d = 0; /* Value is s * pow(10,d) */ - int nDigit = 0; /* Number of digits processed */ - int eType = 1; /* 1: pure integer, 2+: fractional */ - - *pResult = 0.0; /* Default return value, in case of an error */ - - /* skip leading spaces */ - while( sqlite3Isspace(*z) ) z++; - - /* get sign of significand */ - if( *z=='-' ){ + const unsigned char *z = (const unsigned char*)zIn; + int neg = 0; /* True for a negative value */ + u64 s = 0; /* mantissa */ + int d = 0; /* Value is s * pow(10,d) */ + int mState = 0; /* 1: digit seen 2: fp 4: hard-zero */ + unsigned v; /* Value of a single digit */ + + start_of_text: + if( (v = (unsigned)z[0] - '0')<10 ){ + parse_integer_part: + mState = 1; + s = v; + z++; + while( (v = (unsigned)z[0] - '0')<10 ){ + s = s*10 + v; + z++; + if( s>=(LARGEST_UINT64-9)/10 ){ + mState = 9; + while( sqlite3Isdigit(z[0]) ){ z++; d++; } + break; + } + } + }else if( z[0]=='-' ){ neg = 1; z++; - }else if( *z=='+' ){ + if( (v = (unsigned)z[0] - '0')<10 ) goto parse_integer_part; + }else if( z[0]=='+' ){ z++; - } - - /* copy max significant digits to significand */ - while( sqlite3Isdigit(*z) ){ - s = s*10 + (*z - '0'); - z++; nDigit++; - if( s>=((LARGEST_INT64-9)/10) ){ - /* skip non-significant significand digits - ** (increase exponent by d to shift decimal left) */ - while( sqlite3Isdigit(*z) ){ z++; d++; } - } + if( (v = (unsigned)z[0] - '0')<10 ) goto parse_integer_part; + }else if( sqlite3Isspace(z[0]) ){ + do{ z++; }while( sqlite3Isspace(z[0]) ); + goto start_of_text; + }else{ + s = 0; } /* if decimal point is present */ if( *z=='.' ){ z++; - eType++; - /* copy digits from after decimal to significand - ** (decrease exponent by d to shift decimal right) */ - while( sqlite3Isdigit(*z) ){ - if( s<((LARGEST_INT64-9)/10) ){ - s = s*10 + (*z - '0'); - d--; - nDigit++; - } - z++; + if( sqlite3Isdigit(z[0]) ){ + mState |= 1; + do{ + if( s<(LARGEST_UINT64-9)/10 ){ + s = s*10 + z[0] - '0'; + d--; + }else{ + mState = 11; + } + }while( sqlite3Isdigit(*++z) ); + }else if( mState==0 ){ + *pResult = 0.0; + return 0; } + mState |= 2; + }else if( mState==0 ){ + *pResult = 0.0; + return 0; } /* if exponent is present */ if( *z=='e' || *z=='E' ){ - int esign = 1; /* sign of exponent */ + int esign; z++; - eType++; /* get sign of exponent */ if( *z=='-' ){ esign = -1; z++; - }else if( *z=='+' ){ - z++; + }else{ + esign = +1; + if( *z=='+' ){ + z++; + } } /* copy digits to exponent */ - if( sqlite3Isdigit(*z) ){ - int exp = *z - '0'; + if( (v = (unsigned)z[0] - '0')<10 ){ + int exp = v; z++; - while( sqlite3Isdigit(*z) ){ - exp = exp<10000 ? (exp*10 + (*z - '0')) : 10000; + mState |= 2; + while( (v = (unsigned)z[0] - '0')<10 ){ + exp = exp<10000 ? (exp*10 + v) : 10000; z++; } d += esign*exp; }else{ - eType = -1; + z--; /* Leave z[0] at 'e' or '+' or '-', + ** so that the return is 0 or -1 */ } } - /* skip trailing spaces */ - while( sqlite3Isspace(*z) ) z++; - - /* Zero is a special case */ + /* Convert s*pow(10,d) into real */ if( s==0 ){ - *pResult = neg ? -0.0 : +0.0; + *pResult = 0.0; + mState |= 4; }else{ *pResult = sqlite3Fp10Convert2(s,d); - if( neg ) *pResult = -*pResult; - assert( !sqlite3IsNaN(*pResult) ); } + if( neg ) *pResult = -*pResult; + assert( !sqlite3IsNaN(*pResult) ); /* return true if number and no extra non-whitespace characters after */ - if( z[0]==0 && nDigit>0 ){ - return eType; - }else if( eType>=2 && nDigit>0 ){ - return -1; - }else{ - return 0; + if( z[0]==0 ){ + return mState; + } + if( sqlite3Isspace(z[0]) ){ + do{ z++; }while( sqlite3Isspace(*z) ); + if( z[0]==0 ){ + return mState; + } } + return 0xfffffff0 | mState; #else - return !sqlite3Atoi64(z, pResult, strlen(z), SQLITE_UTF8); + return sqlite3Atoi64(z, pResult, strlen(z), SQLITE_UTF8)==0; #endif /* SQLITE_OMIT_FLOATING_POINT */ } -#if defined(_MSC_VER) -#pragma warning(default : 4756) -#endif /* ** Digit pairs used to convert a U64 or I64 into text, two digits @@ -36916,6 +37366,69 @@ static const union { "90919293949596979899" }; +/* +** ARMv6, ARMv7, PPC32 are known to not support hardware u64 division. +*/ +#if (defined(__arm__) && !defined(__aarch64__)) || \ + (defined(__ppc__) && !defined(__ppc64__)) +# define SQLITE_AVOID_U64_DIVIDE 1 +#endif + +#ifdef SQLITE_AVOID_U64_DIVIDE +/* +** Render an unsigned 64-bit integer as text onto the end of a 2-byte +** aligned buffer that is SQLITE_U64_DIGIT+1 bytes long. The last byte +** of the buffer will be filled with a \000 byte. +** +** Return the index into the buffer of the first byte. +** +** This routine is used on platforms where u64-division is slow because +** it is not available in hardware and has to be emulated in software. +** It seeks to minimize the number of u64 divisions and use u32 divisions +** instead. It is slower on platforms that have hardware u64 division, +** but much faster on platforms that do not. +*/ +static int sqlite3UInt64ToText(u64 v, char *zOut){ + u32 x32, kk; + int i; + zOut[SQLITE_U64_DIGITS] = 0; + i = SQLITE_U64_DIGITS; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[0]) ); + assert( TWO_BYTE_ALIGNMENT(zOut) ); + while( (v>>32)!=0 ){ + u32 y, x0, x1, y0, y1; + x32 = v % 100000000; + v = v / 100000000; + y = x32 % 10000; + x32 /= 10000; + x1 = x32 / 100; + x0 = x32 % 100; + y1 = y / 100; + y0 = y % 100; + assert( i>=8 ); + i -= 8; + *(u16*)(&zOut[i]) = *(u16*)&sqlite3DigitPairs.a[x1*2]; + *(u16*)(&zOut[i+2]) = *(u16*)&sqlite3DigitPairs.a[x0*2]; + *(u16*)(&zOut[i+4]) = *(u16*)&sqlite3DigitPairs.a[y1*2]; + *(u16*)(&zOut[i+6]) = *(u16*)&sqlite3DigitPairs.a[y0*2]; + } + x32 = v; + while( x32>=10 ){ + kk = x32 % 100; + x32 = x32 / 100; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk*2]) ); + assert( i>=2 ); + i -= 2; + assert( TWO_BYTE_ALIGNMENT(&zOut[i]) ); + *(u16*)(&zOut[i]) = *(u16*)&sqlite3DigitPairs.a[kk*2]; + } + if( x32 ){ + assert( i>0 ); + zOut[--i] = x32 + '0'; + } + return i; +} +#endif /* defined(SQLITE_AVOID_U64_DIVIDE) */ /* ** Render an signed 64-bit integer as text. Store the result in zOut[] and @@ -36929,7 +37442,7 @@ SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; union { - char a[23]; + char a[SQLITE_U64_DIGITS+1]; u16 forceAlignment; } u; if( v>0 ){ @@ -36941,6 +37454,9 @@ SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ }else{ x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; } +#ifdef SQLITE_AVOID_U64_DIVIDE + i = sqlite3UInt64ToText(x, u.a); +#else i = sizeof(u.a)-1; u.a[i] = 0; while( x>=10 ){ @@ -36954,6 +37470,7 @@ SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ if( x ){ u.a[--i] = x + '0'; } +#endif /* SQLITE_AVOID_U64_DIVIDE */ if( v<0 ) u.a[--i] = '-'; memcpy(zOut, &u.a[i], sizeof(u.a)-i); return sizeof(u.a)-1-i; @@ -37010,8 +37527,8 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc int incr; u64 u = 0; int neg = 0; /* assume positive */ - int i; - int c = 0; + int i, j; + unsigned int c = 0; int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ int rc; /* Baseline return code */ const char *zStart; @@ -37039,8 +37556,8 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc } zStart = zNum; while( zNum='0' && c<='9'; i+=incr){ - u = u*10 + c - '0'; + for(i=0; &zNum[i]19*incr ? 1 : compare2pow63(zNum, incr); - if( c<0 ){ + j = i>19*incr ? 1 : compare2pow63(zNum, incr); + if( j<0 ){ /* zNum is less than 9223372036854775808 so it fits */ assert( u<=LARGEST_INT64 ); return rc; }else{ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; - if( c>0 ){ + if( j>0 ){ /* zNum is greater than 9223372036854775808 so it overflows */ return 2; }else{ @@ -37273,37 +37790,43 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou /* Extract significant digits, start at the right-most slot in p->zBuf ** and working back to the right. "i" keeps track of the next slot in ** which to store a digit. */ - i = sizeof(p->zBuf)-1; - zBuf = p->zBuf; + assert( sizeof(p->zBuf)==SQLITE_U64_DIGITS+1 ); assert( v>0 ); + zBuf = p->zBuf; +#ifdef SQLITE_AVOID_U64_DIVIDE + i = sqlite3UInt64ToText(v, zBuf); +#else + i = SQLITE_U64_DIGITS; while( v>=10 ){ int kk = (v%100)*2; assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); - assert( TWO_BYTE_ALIGNMENT(&zBuf[i-1]) ); - *(u16*)(&zBuf[i-1]) = *(u16*)&sqlite3DigitPairs.a[kk]; + assert( TWO_BYTE_ALIGNMENT(&zBuf[i]) ); + assert( i-2>=0 ); + *(u16*)(&zBuf[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; i -= 2; v /= 100; } if( v ){ assert( v<10 ); - zBuf[i--] = v + '0'; + assert( i>0 ); + zBuf[--i] = v + '0'; } - assert( i>=0 && izBuf)-1 ); - n = sizeof(p->zBuf) - 1 - i; /* Total number of digits extracted */ +#endif /* SQLITE_AVOID_U64_DIVIDE */ + assert( i>=0 && i0 ); - assert( nzBuf) ); - testcase( n==sizeof(p->zBuf)-1 ); + assert( n<=SQLITE_U64_DIGITS ); p->iDP = n + exp; if( iRound<=0 ){ iRound = p->iDP - iRound; - if( iRound==0 && zBuf[i+1]>='5' ){ + if( iRound==0 && zBuf[i]>='5' ){ iRound = 1; - zBuf[i--] = '0'; + zBuf[--i] = '0'; n++; p->iDP++; } } - z = &zBuf[i+1]; /* z points to the first digit */ + z = &zBuf[i]; /* z points to the first digit */ if( iRound>0 && (iRoundmxRound) ){ if( iRound>mxRound ) iRound = mxRound; if( iRound==17 ){ @@ -37330,7 +37853,7 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou int jj, kk; u64 v2; assert( z[0]!='0' ); - for(jj=14; z[jj-1]=='0'; jj--){} + for(jj=13; z[jj-1]=='0'; jj--){} v2 = z[0] - '0'; for(kk=1; kk=r[P1]"), /* 59 */ "ElseEq" OpHelp(""), - /* 60 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), - /* 61 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 62 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 63 */ "IncrVacuum" OpHelp(""), - /* 64 */ "VNext" OpHelp(""), - /* 65 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), - /* 66 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), - /* 67 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), - /* 68 */ "Return" OpHelp(""), - /* 69 */ "EndCoroutine" OpHelp(""), - /* 70 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 71 */ "Halt" OpHelp(""), - /* 72 */ "Integer" OpHelp("r[P2]=P1"), - /* 73 */ "Int64" OpHelp("r[P2]=P4"), - /* 74 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 75 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), - /* 76 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 77 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 78 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 79 */ "Variable" OpHelp("r[P2]=parameter(P1)"), - /* 80 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 81 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 82 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 83 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 84 */ "FkCheck" OpHelp(""), - /* 85 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 86 */ "CollSeq" OpHelp(""), - /* 87 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 88 */ "RealAffinity" OpHelp(""), - /* 89 */ "Cast" OpHelp("affinity(r[P1])"), - /* 90 */ "Permutation" OpHelp(""), - /* 91 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 92 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 93 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), - /* 94 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 95 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), - /* 96 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), - /* 97 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 98 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 99 */ "Count" OpHelp("r[P2]=count()"), - /* 100 */ "ReadCookie" OpHelp(""), - /* 101 */ "SetCookie" OpHelp(""), - /* 102 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 60 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 61 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 62 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 63 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 64 */ "IncrVacuum" OpHelp(""), + /* 65 */ "VNext" OpHelp(""), + /* 66 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), + /* 67 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 68 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 69 */ "Return" OpHelp(""), + /* 70 */ "EndCoroutine" OpHelp(""), + /* 71 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 72 */ "Halt" OpHelp(""), + /* 73 */ "Integer" OpHelp("r[P2]=P1"), + /* 74 */ "Int64" OpHelp("r[P2]=P4"), + /* 75 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 76 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), + /* 77 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 78 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 79 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 80 */ "Variable" OpHelp("r[P2]=parameter(P1)"), + /* 81 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 82 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 83 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 84 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 85 */ "FkCheck" OpHelp(""), + /* 86 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 87 */ "CollSeq" OpHelp(""), + /* 88 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 89 */ "RealAffinity" OpHelp(""), + /* 90 */ "Cast" OpHelp("affinity(r[P1])"), + /* 91 */ "Permutation" OpHelp(""), + /* 92 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 93 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 94 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 95 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 96 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), + /* 97 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 98 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 99 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 100 */ "Count" OpHelp("r[P2]=count()"), + /* 101 */ "ReadCookie" OpHelp(""), + /* 102 */ "SetCookie" OpHelp(""), /* 103 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 104 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 105 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 162 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 163 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 164 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 165 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 166 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 167 */ "Expire" OpHelp(""), - /* 168 */ "CursorLock" OpHelp(""), - /* 169 */ "CursorUnlock" OpHelp(""), - /* 170 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 171 */ "VBegin" OpHelp(""), - /* 172 */ "VCreate" OpHelp(""), - /* 173 */ "VDestroy" OpHelp(""), - /* 174 */ "VOpen" OpHelp(""), - /* 175 */ "VCheck" OpHelp(""), - /* 176 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), - /* 177 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 178 */ "VRename" OpHelp(""), - /* 179 */ "Pagecount" OpHelp(""), - /* 180 */ "MaxPgcnt" OpHelp(""), - /* 181 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), - /* 182 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), - /* 183 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), - /* 184 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), - /* 185 */ "Trace" OpHelp(""), - /* 186 */ "CursorHint" OpHelp(""), - /* 187 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), - /* 188 */ "Noop" OpHelp(""), - /* 189 */ "Explain" OpHelp(""), - /* 190 */ "Abortable" OpHelp(""), + /* 155 */ "DropIndex" OpHelp(""), + /* 156 */ "DropTrigger" OpHelp(""), + /* 157 */ "IntegrityCk" OpHelp(""), + /* 158 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 159 */ "Param" OpHelp(""), + /* 160 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 161 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 162 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 163 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 164 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 165 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 166 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 167 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 168 */ "Expire" OpHelp(""), + /* 169 */ "CursorLock" OpHelp(""), + /* 170 */ "CursorUnlock" OpHelp(""), + /* 171 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 172 */ "VBegin" OpHelp(""), + /* 173 */ "VCreate" OpHelp(""), + /* 174 */ "VDestroy" OpHelp(""), + /* 175 */ "VOpen" OpHelp(""), + /* 176 */ "VCheck" OpHelp(""), + /* 177 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 178 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 179 */ "VRename" OpHelp(""), + /* 180 */ "Pagecount" OpHelp(""), + /* 181 */ "MaxPgcnt" OpHelp(""), + /* 182 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 183 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), + /* 184 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), + /* 185 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 186 */ "Trace" OpHelp(""), + /* 187 */ "CursorHint" OpHelp(""), + /* 188 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 189 */ "Noop" OpHelp(""), + /* 190 */ "Explain" OpHelp(""), + /* 191 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -55615,10 +56139,10 @@ SQLITE_API int sqlite3_deserialize( if( rc ) goto end_deserialize; db->init.iDb = (u8)iDb; db->init.reopenMemdb = 1; - rc = sqlite3_step(pStmt); + sqlite3_step(pStmt); db->init.reopenMemdb = 0; - if( rc!=SQLITE_DONE ){ - rc = SQLITE_ERROR; + rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK ){ goto end_deserialize; } p = memdbFromDbSchema(db, zSchema); @@ -55639,7 +56163,6 @@ SQLITE_API int sqlite3_deserialize( } end_deserialize: - sqlite3_finalize(pStmt); if( pData && (mFlags & SQLITE_DESERIALIZE_FREEONCLOSE)!=0 ){ sqlite3_free(pData); } @@ -68028,7 +68551,7 @@ static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ /* ** Compute a hash on a page number. The resulting hash value must land -** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances +** between 0 and (HASHTABLE_NSLOT-1). The walNextHash() function advances ** the hash to the next value in the event of a collision. */ static int walHash(u32 iPage){ @@ -68236,7 +68759,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - sLoc.aPgno[idx-1] = iPage; + sLoc.aPgno[(idx-1)&(HASHTABLE_NPAGE-1)] = iPage; AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -70484,7 +71007,10 @@ static int walFindFrame( SEH_INJECT_FAULT; while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){ + if( iFrame<=iLast + && iFrame>=pWal->minFrame + && sLoc.aPgno[(iH-1)&(HASHTABLE_NPAGE-1)]==pgno + ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -73859,7 +74385,7 @@ static void btreeParseCellPtr( CellInfo *pInfo /* Fill in this structure */ ){ u8 *pIter; /* For scanning through pCell */ - u32 nPayload; /* Number of bytes of cell payload */ + u64 nPayload; /* Number of bytes of cell payload */ u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -73881,6 +74407,7 @@ static void btreeParseCellPtr( do{ nPayload = (nPayload<<7) | (*++pIter & 0x7f); }while( (*pIter)>=0x80 && pIternKey = *(i64*)&iKey; - pInfo->nPayload = nPayload; + pInfo->nPayload = (u32)nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); - assert( nPayload>=0 ); assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits @@ -85595,32 +86121,36 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem *pMem){ } /* -** Invoke sqlite3AtoF() on the text value of pMem and return the -** double result. If sqlite3AtoF() returns an error code, write -** that code into *pRC if (*pRC)!=NULL. +** This routine implements the uncommon and slower path for +** sqlite3MemRealValueRC() that has to deal with input strings +** that are not UTF8 or that are not zero-terminated. It is +** broken out into a separate no-inline routine so that the +** main sqlite3MemRealValueRC() routine can avoid unnecessary +** stack pushes. ** -** The caller must ensure that pMem->db!=0 and that pMem is in -** mode MEM_Str or MEM_Blob. +** A text->float translation of pMem->z is written into *pValue. +** +** Result code invariants: +** +** rc==0 => ERROR: Input string not well-formed, or OOM +** rc<0 => Some prefix of the input is well-formed +** rc>0 => All of the input is well-formed +** (rc&2)==0 => The number is expressed as an integer, with no +** decimal point or eNNN suffix. */ -SQLITE_PRIVATE SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem *pMem, int *pRC){ - double val = (double)0; - int rc = 0; - assert( pMem->db!=0 ); - assert( pMem->flags & (MEM_Str|MEM_Blob) ); - if( pMem->z==0 ){ - /* no-op */ - }else if( pMem->enc==SQLITE_UTF8 - && ((pMem->flags & MEM_Term)!=0 || sqlite3VdbeMemZeroTerminateIfAble(pMem)) - ){ - rc = sqlite3AtoF(pMem->z, &val); - }else if( pMem->n==0 ){ - /* no-op */ - }else if( pMem->enc==SQLITE_UTF8 ){ +static SQLITE_NOINLINE int sqlite3MemRealValueRCSlowPath( + Mem *pMem, + double *pValue +){ + int rc = SQLITE_OK; + *pValue = 0.0; + if( pMem->enc==SQLITE_UTF8 ){ char *zCopy = sqlite3DbStrNDup(pMem->db, pMem->z, pMem->n); if( zCopy ){ - rc = sqlite3AtoF(zCopy, &val); + rc = sqlite3AtoF(zCopy, pValue); sqlite3DbFree(pMem->db, zCopy); } + return rc; }else{ int n, i, j; char *zCopy; @@ -85643,13 +86173,56 @@ SQLITE_PRIVATE SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem *pMem, int *pRC) } assert( j<=n/2 ); zCopy[j] = 0; - rc = sqlite3AtoF(zCopy, &val); + rc = sqlite3AtoF(zCopy, pValue); if( idb, zCopy); } + return rc; } - if( pRC ) *pRC = rc; - return val; +} + +/* +** Invoke sqlite3AtoF() on the text value of pMem. Write the +** translation of the text input into *pValue. +** +** The caller must ensure that pMem->db!=0 and that pMem is in +** mode MEM_Str or MEM_Blob. +** +** Result code invariants: +** +** rc==0 => ERROR: Input string not well-formed, or OOM +** rc<0 => Some prefix of the input is well-formed +** rc>0 => All of the input is well-formed +** (rc&2)==0 => The number is expressed as an integer, with no +** decimal point or eNNN suffix. +*/ +SQLITE_PRIVATE int sqlite3MemRealValueRC(Mem *pMem, double *pValue){ + testcase( pMem->db==0 ); + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + if( pMem->z==0 ){ + *pValue = 0.0; + return 0; + }else if( pMem->enc==SQLITE_UTF8 + && ((pMem->flags & MEM_Term)!=0 || sqlite3VdbeMemZeroTerminateIfAble(pMem)) + ){ + return sqlite3AtoF(pMem->z, pValue); + }else if( pMem->n==0 ){ + *pValue = 0.0; + return 0; + }else{ + return sqlite3MemRealValueRCSlowPath(pMem, pValue); + } +} + +/* +** This routine acts as a bridge from sqlite3VdbeRealValue() to +** sqlite3VdbeRealValueRC, allowing sqlite3VdbeRealValue() to avoid +** stuffing values onto the stack. +*/ +static SQLITE_NOINLINE double sqlite3MemRealValueNoRC(Mem *pMem){ + double r; + (void)sqlite3MemRealValueRC(pMem, &r); + return r; } /* @@ -85668,7 +86241,7 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - return sqlite3MemRealValueRC(pMem, 0); + return sqlite3MemRealValueNoRC(pMem); }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; @@ -85792,8 +86365,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - pMem->u.r = sqlite3MemRealValueRC(pMem, &rc); - if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + rc = sqlite3MemRealValueRC(pMem, &pMem->u.r); + if( ((rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<2) || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; @@ -89154,6 +89727,10 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = pOp->p4.pTab->zName; break; } + case P4_INDEX: { + zP4 = pOp->p4.pIdx->zName; + break; + } case P4_SUBRTNSIG: { SubrtnSig *pSig = pOp->p4.pSubrtnSig; sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff); @@ -92544,6 +93121,223 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } } +/* +** Helper function for vdbeIsMatchingIndexKey(). Return true if column +** iCol should be ignored when comparing a record with a record from +** an index on disk. The field should be ignored if: +** +** * the corresponding bit in mask is set, and +** * either: +** - bIntegrity is false, or +** - the two Mem values are both real values that differ by +** BTREE_ULPDISTORTION or fewer ULPs. +*/ +static int vdbeSkipField( + Bitmask mask, /* Mask of indexed expression fields */ + int iCol, /* Column of index being considered */ + Mem *pMem1, /* Expected index value */ + Mem *pMem2, /* Actual indexed value */ + int bIntegrity /* True if running PRAGMA integrity_check */ +){ +#define BTREE_ULPDISTORTION 2 + if( iCol>=BMS || (mask & MASKBIT(iCol))==0 ) return 0; + if( bIntegrity==0 ) return 1; + if( (pMem1->flags & MEM_Real) && (pMem2->flags & MEM_Real) ){ + u64 m1, m2; + memcpy(&m1,&pMem1->u.r,8); + memcpy(&m2,&pMem2->u.r,8); + if( (m1pKeyInfo->enc; + mem.db = p->pKeyInfo->db; + nRec = sqlite3BtreePayloadSize(pCur); + if( nRec>0x7fffffff ){ + return SQLITE_CORRUPT_BKPT; + } + + /* Allocate 5 extra bytes at the end of the buffer. This allows the + ** getVarint32() call below to read slightly past the end of the buffer + ** if the record is corrupt. */ + aRec = sqlite3MallocZero(nRec+5); + if( aRec==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3BtreePayload(pCur, 0, nRec, aRec); + } + + if( rc==SQLITE_OK ){ + u32 szHdr = 0; /* Size of record header in bytes */ + u32 idxHdr = 0; /* Current index in header */ + + idxHdr = getVarint32(aRec, szHdr); + if( szHdr>98307 ){ + rc = SQLITE_CORRUPT; + }else{ + int res = 0; /* Result of this function call */ + u32 idxRec = szHdr; /* Index of next field in record body */ + int ii = 0; /* Iterator variable */ + + int nCol = p->pKeyInfo->nAllField; + for(ii=0; ii=szHdr ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + idxHdr += getVarint32(&aRec[idxHdr], iSerial); + nSerial = sqlite3VdbeSerialTypeLen(iSerial); + if( (idxRec+nSerial)>nRec ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3VdbeSerialGet(&aRec[idxRec], iSerial, &mem); + if( vdbeSkipField(mask, ii, &p->aMem[ii], &mem, bInt)==0 ){ + res = sqlite3MemCompare(&mem, &p->aMem[ii], p->pKeyInfo->aColl[ii]); + if( res!=0 ) break; + } + } + idxRec += sqlite3VdbeSerialTypeLen(iSerial); + } + + *piRes = res; + } + } + + sqlite3_free(aRec); + return rc; +} + +/* +** This is called when the record in (*p) should be found in the index +** opened by cursor pCur, but was not. This may happen as part of a DELETE +** operation or an integrity check. +** +** One reason that an exact match was not found may be the EIIB bug - that +** a text-to-float conversion may have caused a real value in record (*p) +** to be slightly different from its counterpart on disk. This function +** attempts to find the right index record. If it does find the right +** record, it leaves *pCur pointing to it and sets (*pRes) to 0 before +** returning. Otherwise, (*pRes) is set to non-zero and an SQLite error +** code returned. +** +** The algorithm used to find the correct record is: +** +** * Scan up to BTREE_FDK_RANGE entries either side of the current entry. +** If parameter bIntegrity is false, then all fields that are indexed +** expressions or virtual table columns are omitted from the comparison. +** If bIntegrity is true, then small differences in real values in +** such fields are overlooked, but they are not omitted from the comparison +** altogether. +** +** * If the above fails to find an entry and bIntegrity is false, search +** the entire index. +*/ +SQLITE_PRIVATE int sqlite3VdbeFindIndexKey( + BtCursor *pCur, + Index *pIdx, + UnpackedRecord *p, + int *pRes, + int bIntegrity +){ +#define BTREE_FDK_RANGE 10 + int nStep = 0; + int res = 1; + int rc = SQLITE_OK; + int ii = 0; + + /* Calculate a mask based on the first 64 columns of the index. The mask + ** bit is set if the corresponding index field is either an expression + ** or a virtual column of the table. */ + Bitmask mask = 0; + for(ii=0; iinColumn, BMS); ii++){ + int iCol = pIdx->aiColumn[ii]; + if( (iCol==XN_EXPR) + || (iCol>=0 && (pIdx->pTable->aCol[iCol].colFlags & COLFLAG_VIRTUAL)) + ){ + mask |= MASKBIT(ii); + } + } + + /* If the mask is 0 at this point, then the index contains no expressions + ** or virtual columns. So do not search for a match - return so that the + ** caller may declare the db corrupt immediately. Or, if mask is non-zero, + ** proceed. */ + if( mask!=0 ){ + + /* Move the cursor back BTREE_FDK_RANGE entries. If this hits an EOF, + ** position the cursor at the first entry in the index and set nStep + ** to -1 so that the first loop below scans the entire index. Otherwise, + ** set nStep to BTREE_FDK_RANGE*2 so that the first loop below scans + ** just that many entries. */ + for(ii=0; sqlite3BtreeEof(pCur)==0 && ii=0), or the entire index if (nStep<0). */ + while( sqlite3BtreeCursorIsValidNN(pCur) ){ + for(ii=0; rc==SQLITE_OK && (iiflags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); - rValue = sqlite3MemRealValueRC(pRec, &rc); + rc = sqlite3MemRealValueRC(pRec, &rValue); if( rc<=0 ) return; - if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ + if( (rc&2)==0 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue; @@ -96129,15 +96923,15 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ pMem->u.i = 0; return MEM_Int; } - pMem->u.r = sqlite3MemRealValueRC(pMem, &rc); + rc = sqlite3MemRealValueRC(pMem, &pMem->u.r); if( rc<=0 ){ - if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + if( (rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ pMem->u.i = ix; return MEM_Int; }else{ return MEM_Real; } - }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + }else if( (rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ pMem->u.i = ix; return MEM_Int; } @@ -102278,13 +103072,15 @@ case OP_SorterInsert: { /* in2 */ break; } -/* Opcode: IdxDelete P1 P2 P3 * * +/* Opcode: IdxDelete P1 P2 P3 P4 * ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form ** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. ** +** P4 is a pointer to an Index structure. +** ** Raise an SQLITE_CORRUPT_INDEX error if no matching index entry is found ** and not in writable_schema mode. */ @@ -102309,13 +103105,22 @@ case OP_IdxDelete: { r.aMem = &aMem[pOp->p2]; rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ - rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); - if( rc ) goto abort_due_to_error; - }else if( !sqlite3WritableSchema(db) ){ - rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); - goto abort_due_to_error; + if( res!=0 ){ + rc = sqlite3VdbeFindIndexKey(pCrsr, pOp->p4.pIdx, &r, &res, 0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + if( res!=0 ){ + if( !sqlite3WritableSchema(db) ){ + rc = sqlite3ReportError( + SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); + goto abort_due_to_error; + } + pC->cacheStatus = CACHE_STALE; + pC->seekResult = 0; + break; + } } + rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); + if( rc ) goto abort_due_to_error; assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; pC->seekResult = 0; @@ -102942,6 +103747,58 @@ case OP_IntegrityCk: { sqlite3VdbeChangeEncoding(pIn1, encoding); goto check_for_interrupt; } + +/* Opcode: IFindKey P1 P2 P3 P4 * +** +** This instruction always follows an OP_Found with the same P1, P2 and P3 +** values as this instruction and a non-zero P4 value. The P4 value to +** this opcode is of type P4_INDEX and contains a pointer to the Index +** object of for the index being searched. +** +** This opcode uses sqlite3VdbeFindIndexKey() to search around the current +** cursor location for an index key that exactly matches all fields that +** are not indexed expressions or references to VIRTUAL generated columns, +** and either exactly match or are real numbers that are within 2 ULPs of +** each other if the don't match. +** +** To put it another way, this opcode looks for nearby index entries that +** are very close to the search key, but which might have small differences +** in floating-point values that come via an expression. +** +** If no nearby alternative entry is found in cursor P1, then jump to P2. +** But if a close match is found, fall through. +** +** This opcode is used by PRAGMA integrity_check to help distinguish +** between truely corrupt indexes and expression indexes that are holding +** floating-point values that are off by one or two ULPs. +*/ +case OP_IFindKey: { /* jump, in3 */ + VdbeCursor *pC; + int res; + UnpackedRecord r; + + assert( pOp[-1].opcode==OP_Found ); + assert( pOp[-1].p1==pOp->p1 ); + assert( pOp[-1].p3==pOp->p3 ); + pC = p->apCsr[pOp->p1]; + assert( pOp->p4type==P4_INDEX ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + assert( pC->isTable==0 ); + + memset(&r, 0, sizeof(r)); + r.aMem = &aMem[pOp->p3]; + r.nField = pOp->p4.pIdx->nColumn; + r.pKeyInfo = pC->pKeyInfo; + + rc = sqlite3VdbeFindIndexKey(pC->uc.pCursor, pOp->p4.pIdx, &r, &res, 1); + if( rc || res!=0 ){ + rc = SQLITE_OK; + goto jump_to_p2; + } + pC->nullRow = 0; + break; +}; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* Opcode: RowSetAdd P1 P2 * * * @@ -116954,7 +117811,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_ISNOT: op = (op==TK_IS) ? TK_EQ : TK_NE; p5 = SQLITE_NULLEQ; - /* fall-through */ + /* no break */ deliberate_fall_through case TK_LT: case TK_LE: case TK_GT: @@ -122295,7 +123152,9 @@ SQLITE_PRIVATE void sqlite3AlterDropConstraint( if( !pTab ) return; if( pCons ){ - zArg = sqlite3MPrintf(db, "%.*Q", pCons->n, pCons->z); + char *z = sqlite3NameFromToken(db, pCons); + zArg = sqlite3MPrintf(db, "%Q", z); + sqlite3DbFree(db, z); }else{ int iCol; if( alterFindCol(pParse, pTab, pCol, &iCol) ) return; @@ -124677,6 +125536,16 @@ static void attachFunc( ** from sqlite3_deserialize() to close database db->init.iDb and ** reopen it as a MemDB */ Btree *pNewBt = 0; + + pNew = &db->aDb[db->init.iDb]; + assert( pNew->pBt!=0 ); + if( sqlite3BtreeTxnState(pNew->pBt)!=SQLITE_TXN_NONE + || sqlite3BtreeIsInBackup(pNew->pBt) + ){ + rc = SQLITE_BUSY; + goto attach_error; + } + pVfs = sqlite3_vfs_find("memdb"); if( pVfs==0 ) return; rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNewBt, 0, SQLITE_OPEN_MAIN_DB); @@ -124686,8 +125555,7 @@ static void attachFunc( /* Both the Btree and the new Schema were allocated successfully. ** Close the old db and update the aDb[] slot with the new memdb ** values. */ - pNew = &db->aDb[db->init.iDb]; - if( ALWAYS(pNew->pBt) ) sqlite3BtreeClose(pNew->pBt); + sqlite3BtreeClose(pNew->pBt); pNew->pBt = pNewBt; pNew->pSchema = pNewSchema; }else{ @@ -127953,9 +128821,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, j, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + const char *zColl = sqlite3ColumnColl(&pTab->aCol[i]); assert( jnColumn ); pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; + pPk->azColl[j] = zColl ? zColl : sqlite3StrBINARY; j++; } } @@ -130981,8 +131850,7 @@ SQLITE_PRIVATE void sqlite3RowidConstraint( } /* -** Check to see if pIndex uses the collating sequence pColl. Return -** true if it does and false if it does not. +** Return true if any column of pIndex uses the zColl collation */ #ifndef SQLITE_OMIT_REINDEX static int collationMatch(const char *zColl, Index *pIndex){ @@ -130990,8 +131858,8 @@ static int collationMatch(const char *zColl, Index *pIndex){ assert( zColl!=0 ); for(i=0; inColumn; i++){ const char *z = pIndex->azColl[i]; - assert( z!=0 || pIndex->aiColumn[i]<0 ); - if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ + assert( z!=0 ); + if( 0==sqlite3StrICmp(z, zColl) ){ return 1; } } @@ -130999,73 +131867,39 @@ static int collationMatch(const char *zColl, Index *pIndex){ } #endif -/* -** Recompute all indices of pTab that use the collating sequence pColl. -** If pColl==0 then recompute all indices of pTab. -*/ -#ifndef SQLITE_OMIT_REINDEX -static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ - if( !IsVirtual(pTab) ){ - Index *pIndex; /* An index associated with pTab */ - - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( zColl==0 || collationMatch(zColl, pIndex) ){ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - } - } - } -} -#endif - -/* -** Recompute all indices of all tables in all databases where the -** indices use the collating sequence pColl. If pColl==0 then recompute -** all indices everywhere. -*/ -#ifndef SQLITE_OMIT_REINDEX -static void reindexDatabases(Parse *pParse, char const *zColl){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - sqlite3 *db = pParse->db; /* The database connection */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - - assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ - for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ - assert( pDb!=0 ); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - reindexTable(pParse, pTab, zColl); - } - } -} -#endif - /* ** Generate code for the REINDEX command. ** ** REINDEX -- 1 ** REINDEX -- 2 -** REINDEX ?.? -- 3 -** REINDEX ?.? -- 4 +** REINDEX ?.? -- 3 +** REINDEX ?.? -- 4 +** REINDEX EXPRESSIONS -- 5 ** -** Form 1 causes all indices in all attached databases to be rebuilt. -** Form 2 rebuilds all indices in all databases that use the named +** Form 1 causes all indexes in all attached databases to be rebuilt. +** Form 2 rebuilds all indexes in all databases that use the named ** collating function. Forms 3 and 4 rebuild the named index or all -** indices associated with the named table. +** indexes associated with the named table, respectively. Form 5 +** rebuilds all expression indexes in addition to all collations, +** indexes, or tables named "EXPRESSIONS". +** +** If the name is ambiguous such that it matches two or more of +** forms 2 through 5, then rebuild the union of all matching indexes, +** taken care to avoid rebuilding the same index more than once. */ #ifndef SQLITE_OMIT_REINDEX SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ - CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ - char *z; /* Name of a table or index */ - const char *zDb; /* Name of the database */ - Table *pTab; /* A table in the database */ - Index *pIndex; /* An index associated with pTab */ - int iDb; /* The database index number */ + char *z = 0; /* Name of a table or index or collation */ + const char *zDb = 0; /* Name of the database */ + int iReDb = -1; /* The database index number */ sqlite3 *db = pParse->db; /* The database connection */ Token *pObjName; /* Name of the table or index to be reindexed */ + int bMatch = 0; /* At least one name match */ + const char *zColl = 0; /* Rebuild indexes using this collation */ + Table *pReTab = 0; /* Rebuild all indexes of this table */ + Index *pReIndex = 0; /* Rebuild this index */ + int isExprIdx = 0; /* Rebuild all expression indexes */ + int bAll = 0; /* Rebuild all indexes */ /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -131074,41 +131908,66 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ } if( pName1==0 ){ - reindexDatabases(pParse, 0); - return; + /* rebuild all indexes */ + bMatch = 1; + bAll = 1; }else if( NEVER(pName2==0) || pName2->z==0 ){ - char *zColl; assert( pName1->z ); - zColl = sqlite3NameFromToken(pParse->db, pName1); - if( !zColl ) return; - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - if( pColl ){ - reindexDatabases(pParse, zColl); - sqlite3DbFree(db, zColl); - return; + z = sqlite3NameFromToken(pParse->db, pName1); + if( z==0 ) return; + }else{ + iReDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iReDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iReDb].zDbSName; + } + if( !bAll ){ + if( zDb==0 && sqlite3StrICmp(z, "expressions")==0 ){ + isExprIdx = 1; + bMatch = 1; + } + if( zDb==0 && sqlite3FindCollSeq(db, ENC(db), z, 0)!=0 ){ + zColl = z; + bMatch = 1; + } + if( zColl==0 && (pReTab = sqlite3FindTable(db, z, zDb))!=0 ){ + bMatch = 1; + } + if( zColl==0 && (pReIndex = sqlite3FindIndex(db, z, zDb))!=0 ){ + bMatch = 1; } - sqlite3DbFree(db, zColl); } - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); - if( iDb<0 ) return; - z = sqlite3NameFromToken(db, pObjName); - if( z==0 ) return; - zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; - pTab = sqlite3FindTable(db, z, zDb); - if( pTab ){ - reindexTable(pParse, pTab, 0); - sqlite3DbFree(db, z); - return; + if( bMatch ){ + int iDb; + HashElem *k; + Table *pTab; + Index *pIdx; + Db *pDb; + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + assert( pDb!=0 ); + if( iReDb>=0 && iReDb!=iDb ) continue; + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + if( IsVirtual(pTab) ) continue; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( bAll + || pTab==pReTab + || pIdx==pReIndex + || (isExprIdx && pIdx->bHasExpr) + || (zColl!=0 && collationMatch(zColl,pIdx)) + ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIdx, -1); + } + } /* End loop over indexes of pTab */ + } /* End loop over tables of iDb */ + } /* End loop over databases */ + }else{ + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); } - pIndex = sqlite3FindIndex(db, z, zDb); sqlite3DbFree(db, z); - if( pIndex ){ - iDb = sqlite3SchemaToIndex(db, pIndex->pTable->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - return; - } - sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); + return; } #endif @@ -132755,7 +133614,9 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn + ); + sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -133962,18 +134823,11 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int switch( sqlite3_value_type(pValue) ){ case SQLITE_FLOAT: { - double r1, r2; - const char *zVal; - r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!0.15g", r1); - zVal = sqlite3_str_value(pStr); - if( zVal ){ - sqlite3AtoF(zVal, &r2); - if( r1!=r2 ){ - sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!0.20e", r1); - } - } + /* ,--- Show infinity as 9.0e+999 + ** | + ** | ,--- 17 precision guarantees round-trip + ** v v */ + sqlite3_str_appendf(pStr, "%!0.17g", sqlite3_value_double(pValue)); break; } case SQLITE_INTEGER: { @@ -137656,7 +138510,6 @@ static Trigger *fkActionTrigger( nFrom = sqlite3Strlen30(zFrom); if( action==OE_Restrict ){ - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); SrcList *pSrc; Expr *pRaise; @@ -137667,10 +138520,10 @@ static Trigger *fkActionTrigger( } pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ - assert( pSrc->nSrc==1 ); - pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); - pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + SrcItem *pItem = &pSrc->a[0]; + pItem->zName = sqlite3DbStrDup(db, zFrom); + pItem->fg.fixedSchema = 1; + pItem->u4.pSchema = pTab->pSchema; } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), @@ -137692,7 +138545,10 @@ static Trigger *fkActionTrigger( pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; pStep->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pStep->pSrc ){ - pStep->pSrc->a[0].zName = sqlite3DbStrNDup(db, zFrom, nFrom); + SrcItem *pItem = &pStep->pSrc->a[0]; + pItem->zName = sqlite3DbStrNDup(db, zFrom, nFrom); + pItem->u4.pSchema = pTab->pSchema; + pItem->fg.fixedSchema = 1; } pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); @@ -145781,8 +146637,20 @@ SQLITE_PRIVATE void sqlite3Pragma( pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ /* Verify that an index entry exists for the current table row */ - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); + jmp2 = sqlite3VdbeAddOp3(v, OP_IFindKey, iIdxCur+j, ckUniq, r1); + VdbeCoverage(v); + sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "index %s stores an imprecise floating-point " + "value for row ", pIdx->zName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + integrityCheckResultRow(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ckUniq); + + sqlite3VdbeJumpHere(v, jmp2); sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); @@ -145790,7 +146658,7 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp4 = integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeResolveLabel(v, ckUniq); /* The OP_IdxRowid opcode is an optimized version of OP_Column ** that extracts the rowid off the end of the index record. @@ -155230,6 +156098,7 @@ static SQLITE_NOINLINE void existsToJoin( && !ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) && ALWAYS(p->pSrc!=0) && p->pSrc->nSrcpLimit==0 || p->pLimit->pRight==0) ){ if( pWhere->op==TK_AND ){ Expr *pRight = pWhere->pRight; @@ -155277,7 +156146,6 @@ static SQLITE_NOINLINE void existsToJoin( sqlite3TreeViewSelect(0, p, 0); } #endif - existsToJoin(pParse, p, pSubWhere); } } } @@ -165119,7 +165987,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). */ - if( pWInfo->nLevel>1 ){ + if( pWInfo->nLevel>1 || pTabItem->fg.fromExists ){ int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; @@ -165132,6 +166000,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( for(k=1; k<=nNotReady; k++){ memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); } + + /* Clear the fromExists flag on the OR-optimized table entry so that + ** the calls to sqlite3WhereEnd() do not code early-exits after the + ** first row is visited. The early exit applies to this table's + ** overall loop - including the multiple OR branches and any WHERE + ** conditions not passed to the sub-loops - not to the sub-loops. */ + pOrTab->a[0].fg.fromExists = 0; }else{ pOrTab = pWInfo->pTabList; } @@ -165375,7 +166250,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->op==OP_Return ); pLevel->p2 = sqlite3VdbeCurrentAddr(v); - if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } + if( pWInfo->pTabList!=pOrTab ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -165714,6 +166589,15 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } + if( pLevel->iIdxCur ){ + /* pSubWhere may contain expressions that read from an index on the + ** table on the RHS of the right join. All such expressions first test + ** if the index is pointing at a NULL row, and if so, read from the + ** table cursor instead. So ensure that the index cursor really is + ** pointing at a NULL row here, so that no values are read from it during + ** the scan of the RHS of the RIGHT join below. */ + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } pFrom = &uSrc.sSrc; pFrom->nSrc = 1; pFrom->nAlloc = 1; @@ -170157,11 +171041,16 @@ SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ WhereInfo *pWInfo; if( pWC ){ + int nb; + SrcItem *pItem; + Table *pTab; + Bitmask mAll; + pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pSTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + nb = 1+(pWInfo->pTabList->nSrc+3)/4; + pItem = pWInfo->pTabList->a + p->iTab; + pTab = pItem->pSTab; + mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); sqlite3DebugPrintf(" %12s", @@ -171875,7 +172764,12 @@ static int whereLoopAddBtree( whereLoopOutputAdjust(pWC, pNew, rSize); if( pSrc->fg.isSubquery ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; - pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + /* Do not set btree.pOrderBy for a recursive CTE. In this case + ** the ORDER BY clause does not determine the overall order that + ** rows are emitted from the CTE in. */ + if( (pSrc->u4.pSubq->pSelect->selFlags & SF_Recursive)==0 ){ + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + } }else if( pSrc->fg.fromExists ){ pNew->nOut = 0; } @@ -175262,6 +176156,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ int r1 = pParse->nMem+1; int j, op; + int addrIfNull = 0; /* Init to avoid false-positive compiler warning */ + if( pLevel->iLeftJoin ){ + addrIfNull = sqlite3VdbeAddOp2(v, OP_IfNullRow, pLevel->iIdxCur, r1); + } for(j=0; jiIdxCur, j, r1+j); } @@ -175271,30 +176169,17 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, op==OP_SeekLT); VdbeCoverageIf(v, op==OP_SeekGT); sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + if( pLevel->iLeftJoin ){ + sqlite3VdbeJumpHere(v, addrIfNull); + } } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ } - if( pTabList->a[pLevel->iFrom].fg.fromExists - && (i==pWInfo->nLevel-1 - || pTabList->a[pWInfo->a[i+1].iFrom].fg.fromExists==0) - ){ - /* This is an EXISTS-to-JOIN optimization which is either the - ** inner-most loop, or the inner-most of a group of nested - ** EXISTS-to-JOIN optimization loops. If this loop sees a successful - ** row, it should break out of itself as well as other EXISTS-to-JOIN - ** loops in which is is directly nested. */ - int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ - while( nOutera[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; - nOuter++; - } - testcase( nOuter>0 ); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); - if( nOuter ){ - VdbeComment((v, "EXISTS break %d..%d", i-nOuter, i)); - }else{ - VdbeComment((v, "EXISTS break %d", i)); - } + if( pTabList->a[pLevel->iFrom].fg.fromExists ){ + /* This is an EXISTS-to-JOIN optimization loop. If this loop sees a + ** successful row, it should break out of itself. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); + VdbeComment((v, "EXISTS break %d", i)); } sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ @@ -183481,6 +184366,7 @@ static YYACTIONTYPE yy_reduce( yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0); if( yymsp[-4].minor.yy454 ){ yymsp[-4].minor.yy454->x.pList = pList; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ sqlite3ExprListDelete(pParse->db, pList); } @@ -185668,7 +186554,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ } if( pParse->zErrMsg || (pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE) ){ if( pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + pParse->zErrMsg = sqlite3DbStrDup(db, sqlite3ErrStr(pParse->rc)); } if( (pParse->prepFlags & SQLITE_PREPARE_DONT_LOG)==0 ){ sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); @@ -185749,7 +186635,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( sqlite3_str_append(pStr, " NULL", 5); break; } - /* Fall through */ + /* no break */ deliberate_fall_through } case TK_STRING: case TK_INTEGER: @@ -185813,7 +186699,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( } case TK_SELECT: { iStartIN = 0; - /* fall through */ + /* no break */ deliberate_fall_through } default: { if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); @@ -190939,6 +191825,17 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_ATOF, const char *z, double *p); + ** + ** Test access to the sqlite3AtoF() routine. + */ + case SQLITE_TESTCTRL_ATOF: { + const char *z = va_arg(ap,const char*); + double *pR = va_arg(ap,double*); + rc = sqlite3AtoF(z,pR); + break; + } + #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) ** @@ -192529,7 +193426,16 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define deliberate_fall_through +#if !defined(deliberate_fall_through) +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through +#endif /* ** Macros needed to provide flexible arrays in a portable way @@ -199258,7 +200164,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->aStat[1].nDoc++; } eState = 2; - /* fall through */ + /* no break */ deliberate_fall_through case 2: if( v==0 ){ /* 0x00. Next integer will be a docid. */ @@ -211820,7 +212726,8 @@ struct JsonString { /* Allowed values for JsonString.eErr */ #define JSTRING_OOM 0x01 /* Out of memory */ #define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ -#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ +#define JSTRING_TOODEEP 0x04 /* JSON nested too deep */ +#define JSTRING_ERR 0x08 /* Error already sent to sqlite3_result */ /* The "subtype" set for text JSON values passed through using ** sqlite3_result_subtype() and sqlite3_value_subtype(). @@ -211910,7 +212817,7 @@ struct JsonParse { **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); -static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); +static u32 jsonTranslateBlobToText(JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); @@ -212068,6 +212975,15 @@ static void jsonStringOom(JsonString *p){ jsonStringReset(p); } +/* Report JSON nested too deep +*/ +static void jsonStringTooDeep(JsonString *p){ + p->eErr |= JSTRING_TOODEEP; + assert( p->pCtx!=0 ); + sqlite3_result_error(p->pCtx, "JSON nested too deep", -1); + jsonStringReset(p); +} + /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ @@ -212357,6 +213273,7 @@ static void jsonReturnString( ){ assert( (pParse!=0)==(ctx!=0) ); assert( ctx==0 || ctx==p->pCtx ); + jsonStringTerminate(p); if( p->eErr==0 ){ int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); if( flags & JSON_BLOB ){ @@ -212364,7 +213281,7 @@ static void jsonReturnString( }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); - }else if( jsonStringTerminate(p) ){ + }else{ if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ int rc; pParse->zJson = sqlite3RCStrRef(p->zBuf); @@ -212380,11 +213297,11 @@ static void jsonReturnString( sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); - }else{ - sqlite3_result_error_nomem(p->pCtx); } }else if( p->eErr & JSTRING_OOM ){ sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_TOODEEP ){ + /* error already in p->pCtx */ }else if( p->eErr & JSTRING_MALFORMED ){ sqlite3_result_error(p->pCtx, "malformed JSON", -1); } @@ -212715,11 +213632,11 @@ static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ /* Slow version of jsonBlobAppendNode() that first resizes the ** pParse->aBlob structure. */ -static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static void jsonBlobAppendNode(JsonParse*,u8,u64,const void*); static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( JsonParse *pParse, u8 eType, - u32 szPayload, + u64 szPayload, const void *aPayload ){ if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; @@ -212739,7 +213656,7 @@ static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( static void jsonBlobAppendNode( JsonParse *pParse, /* The JsonParse object under construction */ u8 eType, /* Node type. One of JSONB_* */ - u32 szPayload, /* Number of bytes of payload */ + u64 szPayload, /* Number of bytes of payload */ const void *aPayload /* The payload. Might be NULL */ ){ u8 *a; @@ -213595,12 +214512,8 @@ static int jsonConvertTextToBlob( */ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; + assert( pStr->eErr==0 ); memset(&px, 0, sizeof(px)); - jsonStringTerminate(pStr); - if( pStr->eErr ){ - sqlite3_result_error_nomem(pStr->pCtx); - return; - } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); @@ -213690,7 +214603,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ static u32 jsonTranslateBlobToText( - const JsonParse *pParse, /* the complete parse of the JSON */ + JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ ){ @@ -213872,10 +214785,14 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '['); j = i+n; iEnd = j+sz; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( jeErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, ','); } + pParse->iDepth--; if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, ']'); @@ -213886,10 +214803,14 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '{'); j = i+n; iEnd = j+sz; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( jeErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } + pParse->iDepth--; if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, '}'); @@ -213946,7 +214867,7 @@ static u32 jsonTranslateBlobToPrettyText( u32 i /* Start rendering at this index */ ){ u32 sz, n, j, iEnd; - const JsonParse *pParse = pPretty->pParse; + JsonParse *pParse = pPretty->pParse; JsonString *pOut = pPretty->pOut; n = jsonbPayloadSize(pParse, i, &sz); if( n==0 ){ @@ -213961,6 +214882,9 @@ static u32 jsonTranslateBlobToPrettyText( if( jnIndent++; + if( pPretty->nIndent >= JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToPrettyText(pPretty, j); @@ -213982,6 +214906,10 @@ static u32 jsonTranslateBlobToPrettyText( if( jnIndent++; + if( pPretty->nIndent >= JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } + pParse->iDepth = pPretty->nIndent; while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToText(pParse, j, pOut); @@ -214139,6 +215067,7 @@ static void jsonBlobEdit( u32 nIns /* Bytes of content to insert */ ){ i64 d = (i64)nIns - (i64)nDel; + assert( pParse->nBlob >= (u64)iDel + (u64)nDel ); if( d<0 && d>=(-8) && aIns!=0 && jsonBlobOverwrite(&pParse->aBlob[iDel], aIns, nIns, (int)-d) ){ @@ -214382,7 +215311,8 @@ static int jsonLabelCompare( #define JSON_LOOKUP_ERROR 0xffffffff #define JSON_LOOKUP_NOTFOUND 0xfffffffe #define JSON_LOOKUP_NOTARRAY 0xfffffffd -#define JSON_LOOKUP_PATHERROR 0xfffffffc +#define JSON_LOOKUP_TOODEEP 0xfffffffc +#define JSON_LOOKUP_PATHERROR 0xfffffffb #define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) /* Forward declaration */ @@ -214429,7 +215359,12 @@ static u32 jsonCreateEditSubstructure( pIns->eEdit = pParse->eEdit; pIns->nIns = pParse->nIns; pIns->aIns = pParse->aIns; + pIns->iDepth = pParse->iDepth+1; + if( pIns->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->iDepth--; pParse->oom |= pIns->oom; } return rc; /* Error code only */ @@ -214535,7 +215470,11 @@ static u32 jsonLookupStep( n = jsonbPayloadSize(pParse, v, &sz); if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; assert( j>0 ); + if( ++pParse->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pParse, v, &zPath[i], j); + pParse->iDepth--; if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } @@ -214621,7 +215560,11 @@ static u32 jsonLookupStep( iEnd = j+sz; while( jiDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pParse, j, &zPath[i+1], 0); + pParse->iDepth--; if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } @@ -214953,7 +215896,16 @@ static int jsonFunctionArgToBlob( } /* -** Generate a bad path error. +** Generate a path error. +** +** The specifics of the error are determined by the rc argument. +** +** rc error +** ----------------- ---------------------- +** JSON_LOOKUP_ARRAY "not an array" +** JSON_LOOKUP_TOODEEP "JSON nested too deep" +** JSON_LOOKUP_ERROR "malformed JSON" +** otherwise... "bad JSON path" ** ** If ctx is not NULL then push the error message into ctx and return NULL. ** If ctx is NULL, then return the text of the error message. @@ -214966,6 +215918,10 @@ static char *jsonBadPathError( char *zMsg; if( rc==(int)JSON_LOOKUP_NOTARRAY ){ zMsg = sqlite3_mprintf("not an array element: %Q", zPath); + }else if( rc==(int)JSON_LOOKUP_ERROR ){ + zMsg = sqlite3_mprintf("malformed JSON"); + }else if( rc==(int)JSON_LOOKUP_TOODEEP ){ + zMsg = sqlite3_mprintf("JSON path too deep"); }else{ zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); } @@ -215028,6 +215984,7 @@ static void jsonInsertIntoBlob( p->nIns = ax.nBlob; p->aIns = ax.aBlob; p->delta = 0; + p->iDepth = 0; rc = jsonLookupStep(p, 0, zPath+1, 0); } jsonParseReset(&ax); @@ -215040,11 +215997,7 @@ static void jsonInsertIntoBlob( jsonInsertIntoBlob_patherror: jsonParseFree(p); - if( rc==JSON_LOOKUP_ERROR ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - }else{ - jsonBadPathError(ctx, zPath, rc); - } + jsonBadPathError(ctx, zPath, rc); return; } @@ -215484,10 +216437,8 @@ static void jsonArrayLengthFunc( if( JSON_LOOKUP_ISERROR(i) ){ if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath, 0); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, i); } eErr = 1; i = 0; @@ -215621,11 +216572,8 @@ static void jsonExtractFunc( jsonAppendSeparator(&jx); jsonAppendRawNZ(&jx, "null", 4); } - }else if( j==JSON_LOOKUP_ERROR ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - goto json_extract_error; }else{ - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath, j); goto json_extract_error; } } @@ -215649,6 +216597,7 @@ static void jsonExtractFunc( #define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ #define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ #define JSON_MERGE_OOM 3 /* Out-of-memory condition */ +#define JSON_MERGE_TOODEEP 4 /* Nested too deep */ /* ** RFC-7396 MergePatch for two JSONB blobs. @@ -215700,7 +216649,8 @@ static int jsonMergePatch( JsonParse *pTarget, /* The JSON parser that contains the TARGET */ u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ const JsonParse *pPatch, /* The PATCH */ - u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ + u32 iPatch, /* Index of PATCH in pPatch->aBlob[] */ + u32 iDepth /* Nesting depth */ ){ u8 x; /* Type of a single node */ u32 n, sz=0; /* Return values from jsonbPayloadSize() */ @@ -215809,7 +216759,8 @@ static int jsonMergePatch( /* Algorithm line 12 */ int rc, savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); + if( iDepth>=JSON_MAX_DEPTH ) return JSON_MERGE_TOODEEP; + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue, iDepth+1); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -215830,7 +216781,8 @@ static int jsonMergePatch( pTarget->aBlob[iTEnd+szNew] = 0x00; savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); + if( iDepth>=JSON_MAX_DEPTH ) return JSON_MERGE_TOODEEP; + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue,iDepth+1); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -215861,11 +216813,13 @@ static void jsonPatchFunc( if( pTarget==0 ) return; pPatch = jsonParseFuncArg(ctx, argv[1], 0); if( pPatch ){ - rc = jsonMergePatch(pTarget, 0, pPatch, 0); + rc = jsonMergePatch(pTarget, 0, pPatch, 0, 0); if( rc==JSON_MERGE_OK ){ jsonReturnParse(ctx, pTarget); }else if( rc==JSON_MERGE_OOM ){ sqlite3_result_error_nomem(ctx); + }else if( rc==JSON_MERGE_TOODEEP ){ + sqlite3_result_error(ctx, "JSON nested too deep", -1); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -215953,10 +216907,8 @@ static void jsonRemoveFunc( if( JSON_LOOKUP_ISERROR(rc) ){ if( rc==JSON_LOOKUP_NOTFOUND ){ continue; /* No-op */ - }else if( rc==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath, rc); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, rc); } goto json_remove_done; } @@ -216053,10 +217005,8 @@ static void jsonTypeFunc( if( JSON_LOOKUP_ISERROR(i) ){ if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath, 0); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, i); } goto json_type_done; } @@ -216314,7 +217264,8 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ pStr->pCtx = ctx; - jsonAppendChar(pStr, ']'); + jsonAppendRawNZ(pStr, "]", 2); + jsonStringTrimOneChar(pStr); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -216437,7 +217388,8 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - jsonAppendChar(pStr, '}'); + jsonAppendRawNZ(pStr, "}", 2); + jsonStringTrimOneChar(pStr); pStr->pCtx = ctx; if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); @@ -218181,7 +219133,17 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; assert( pRtree->nCursor==0 ); nodeBlobReset(pRtree); - assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); + if( pRtree->nNodeRef ){ + int i; + assert( pRtree->bCorrupt ); + for(i=0; iaHash[i] ){ + RtreeNode *pNext = pRtree->aHash[i]->pNext; + sqlite3_free(pRtree->aHash[i]); + pRtree->aHash[i] = pNext; + } + } + } sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -219473,7 +220435,7 @@ static int AdjustTree( int iCell; cnt++; - if( NEVER(cnt>100) ){ + if( cnt>100 ){ RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -219831,15 +220793,6 @@ static int SplitNode( rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRight); - pRight = 0; - } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pLeft); - pLeft = 0; - } - splitnode_out: nodeRelease(pRtree, pRight); nodeRelease(pRtree, pLeft); @@ -220024,7 +220977,7 @@ static int rtreeInsertCell( rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); - if( ALWAYS(rc==SQLITE_OK) ){ + if( rc==SQLITE_OK ){ if( iHeight==0 ){ rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); }else{ @@ -232747,9 +233700,10 @@ static int sessionVarintGet(const u8 *aBuf, int *piVal){ ** Return the number of bytes read. */ static int sessionVarintGetSafe(const u8 *aBuf, int nBuf, int *piVal){ - u8 aCopy[5]; + u8 aCopy[9]; const u8 *aRead = aBuf; - if( nBuf<5 ){ + memset(aCopy, 0, sizeof(aCopy)); + if( nBuf> 0) & 0xFF; } +/* +** Write a double value to the buffer aBuf[]. +*/ +static void sessionPutDouble(u8 *aBuf, double r){ + /* TODO: SQLite does something special to deal with mixed-endian + ** floating point values (e.g. ARM7). This code probably should + ** too. */ + u64 i; + assert( sizeof(double)==8 && sizeof(u64)==8 ); + memcpy(&i, &r, 8); + sessionPutI64(aBuf, i); +} + /* ** This function is used to serialize the contents of value pValue (see ** comment titled "RECORD FORMAT" above). @@ -232821,16 +233788,13 @@ static int sessionSerializeValue( /* TODO: SQLite does something special to deal with mixed-endian ** floating point values (e.g. ARM7). This code probably should ** too. */ - u64 i; if( eType==SQLITE_INTEGER ){ - i = (u64)sqlite3_value_int64(pValue); + u64 i = (u64)sqlite3_value_int64(pValue); + sessionPutI64(&aBuf[1], i); }else{ - double r; - assert( sizeof(double)==8 && sizeof(u64)==8 ); - r = sqlite3_value_double(pValue); - memcpy(&i, &r, 8); + double r = sqlite3_value_double(pValue); + sessionPutDouble(&aBuf[1], r); } - sessionPutI64(&aBuf[1], i); } nByte = 9; break; @@ -233020,10 +233984,11 @@ static int sessionSerialLen(const u8 *a){ int n; assert( a!=0 ); e = *a; - if( e==0 || e==0xFF ) return 1; - if( e==SQLITE_NULL ) return 1; if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; - return sessionVarintGet(&a[1], &n) + 1 + n; + if( e==SQLITE_TEXT || e==SQLITE_BLOB ){ + return sessionVarintGet(&a[1], &n) + 1 + n; + } + return 1; } /* @@ -233046,17 +234011,17 @@ static unsigned int sessionChangeHash( u8 *a = aRecord; /* Used to iterate through change record */ for(i=0; inCol; i++){ - int eType = *a; int isPK = pTab->abPK[i]; if( bPkOnly && isPK==0 ) continue; - assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT - || eType==SQLITE_TEXT || eType==SQLITE_BLOB - || eType==SQLITE_NULL || eType==0 - ); - if( isPK ){ - a++; + int eType = *a++; + + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_TEXT || eType==SQLITE_BLOB + || eType==SQLITE_NULL || eType==0 + ); + h = sessionHashAppendType(h, eType); if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ h = sessionHashAppendI64(h, sessionGetI64(a)); @@ -233762,9 +234727,7 @@ static void sessionUpdateOneChange( case SQLITE_FLOAT: { double rVal = sqlite3_column_double(pDflt, iField); - i64 iVal = 0; - memcpy(&iVal, &rVal, sizeof(rVal)); - sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + sessionPutDouble(&pNew->aRecord[pNew->nRecord], rVal); pNew->nRecord += 8; break; } @@ -235021,15 +235984,14 @@ static void sessionAppendCol( int eType = sqlite3_column_type(pStmt, iCol); sessionAppendByte(p, (u8)eType, pRc); if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - sqlite3_int64 i; u8 aBuf[8]; if( eType==SQLITE_INTEGER ){ - i = sqlite3_column_int64(pStmt, iCol); + sqlite3_int64 i = sqlite3_column_int64(pStmt, iCol); + sessionPutI64(aBuf, i); }else{ double r = sqlite3_column_double(pStmt, iCol); - memcpy(&i, &r, 8); + sessionPutDouble(aBuf, r); } - sessionPutI64(aBuf, i); sessionAppendBlob(p, aBuf, 8, pRc); } if( eType==SQLITE_BLOB || eType==SQLITE_TEXT ){ @@ -236072,7 +237034,7 @@ static int sessionChangesetBufferRecord( int *pnByte /* OUT: Size of record in bytes */ ){ int rc = SQLITE_OK; - int nByte = 0; + i64 nByte = 0; int i; for(i=0; rc==SQLITE_OK && iaData[pIn->iNext + nByte++]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int n; - nByte += sessionVarintGet(&pIn->aData[pIn->iNext+nByte], &n); + int nRem = pIn->nData - (pIn->iNext + nByte); + nByte += sessionVarintGetSafe(&pIn->aData[pIn->iNext+nByte], nRem, &n); nByte += n; rc = sessionInputBuffer(pIn, nByte); }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ nByte += 8; + }else if( eType!=0 && eType!=SQLITE_NULL ){ + rc = SQLITE_CORRUPT_BKPT; } } + if( rc==SQLITE_OK && (pIn->iNext+nByte)>pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + } } *pnByte = nByte; return rc; @@ -238070,6 +239038,21 @@ SQLITE_API int sqlite3changeset_apply_strm( ); } +/* +** The parts of the sqlite3_changegroup structure used by the +** sqlite3changegroup_change_xxx() APIs. +*/ +typedef struct ChangeData ChangeData; +struct ChangeData { + SessionTable *pTab; + int bIndirect; + int eOp; + + int nBufAlloc; + SessionBuffer *aBuf; + SessionBuffer record; +}; + /* ** sqlite3_changegroup handle. */ @@ -238081,12 +239064,17 @@ struct sqlite3_changegroup { sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ + ChangeData cd; /* Used by changegroup_change_xxx() APIs. */ }; /* ** This function is called to merge two changes to the same row together as ** part of an sqlite3changeset_concat() operation. A new change object is ** allocated and a pointer to it stored in *ppNew. +** +** Because they have been vetted by sqlite3changegroup_add() or similar, +** both the aRec[] change and the pExist change are safe to use without +** checking for buffer overflows. */ static int sessionChangeMerge( SessionTable *pTab, /* Table structure */ @@ -238227,7 +239215,7 @@ static int sessionChangeMerge( memcpy(aCsr, aRec, nRec); aCsr += nRec; }else{ - if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist, 0,aRec,0) ){ + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist,0,aRec,0) ){ sqlite3_free(pNew); pNew = 0; } @@ -238320,15 +239308,14 @@ static int sessionChangesetExtendRecord( switch( eType ){ case SQLITE_FLOAT: case SQLITE_INTEGER: { - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); - }else{ - double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); - memcpy(&iVal, &rVal, sizeof(i64)); - } if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ - sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + if( eType==SQLITE_INTEGER ){ + sqlite3_int64 iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + }else{ + double rVal = sqlite3_column_double(pTab->pDfltStmt, ii); + sessionPutDouble(&pOut->aBuf[pOut->nBuf], rVal); + } pOut->nBuf += 8; } break; @@ -238399,13 +239386,19 @@ static int sessionChangesetFindTable( int nCol = 0; *ppTab = 0; - sqlite3changeset_pk(pIter, &abPK, &nCol); /* Search the list for an existing table */ for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; } + + if( pIter ){ + sqlite3changeset_pk(pIter, &abPK, &nCol); + }else if( !pTab && !pGrp->db ){ + return SQLITE_OK; + } + /* If one was not found above, create a new table now */ if( !pTab ){ SessionTable **ppNew; @@ -238417,15 +239410,17 @@ static int sessionChangesetFindTable( memset(pTab, 0, sizeof(SessionTable)); pTab->nCol = nCol; pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); + if( nCol>0 ){ + memcpy(pTab->abPK, abPK, nCol); + } pTab->zName = (char*)&pTab->abPK[nCol]; memcpy(pTab->zName, zTab, nTab+1); if( pGrp->db ){ pTab->nCol = 0; rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); + if( rc || pTab->nCol==0 ){ + sqlite3_free(pTab->azCol); sqlite3_free(pTab); return rc; } @@ -238440,7 +239435,7 @@ static int sessionChangesetFindTable( } /* Check that the table is compatible. */ - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + if( pIter && !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ rc = SQLITE_SCHEMA; } @@ -238449,44 +239444,27 @@ static int sessionChangesetFindTable( } /* -** Add the change currently indicated by iterator pIter to the hash table -** belonging to changegroup pGrp. +** Add a single change to the changegroup pGrp. */ static int sessionOneChangeToHash( - sqlite3_changegroup *pGrp, - sqlite3_changeset_iter *pIter, - int bRebase + sqlite3_changegroup *pGrp, /* Changegroup to update */ + SessionTable *pTab, /* Table change pertains to */ + int op, /* One of SQLITE_INSERT, UPDATE, DELETE */ + int bIndirect, /* True to flag change as "indirect" */ + int nCol, /* Number of columns in record(s) */ + u8 *aRec, /* Serialized change record(s) */ + int nRec, /* Size of aRec[] in bytes */ + int bRebase /* True if this is a rebase blob */ ){ int rc = SQLITE_OK; - int nCol = 0; - int op = 0; int iHash = 0; - int bIndirect = 0; SessionChange *pChange = 0; SessionChange *pExist = 0; SessionChange **pp = 0; - SessionTable *pTab = 0; - u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; - int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; assert( nRec>0 ); - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_OK ){ - const char *zTab = 0; - sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); - rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); - } - - if( rc==SQLITE_OK && nColnCol ){ + if( nColnCol ){ SessionBuffer *pBuf = &pGrp->rec; rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); aRec = pBuf->aBuf; @@ -238494,7 +239472,7 @@ static int sessionOneChangeToHash( assert( pGrp->db ); } - if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + if( rc==SQLITE_OK && sessionGrowHash(0, pGrp->bPatch, pTab) ){ rc = SQLITE_NOMEM; } @@ -238502,12 +239480,12 @@ static int sessionOneChangeToHash( /* Search for existing entry. If found, remove it from the hash table. ** Code below may link it back in. */ iHash = sessionChangeHash( - pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange + pTab, (pGrp->bPatch && op==SQLITE_DELETE), aRec, pTab->nChange ); for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; - if( pIter->bPatchset ){ + if( pGrp->bPatch ){ bPkOnly1 = (*pp)->op==SQLITE_DELETE; bPkOnly2 = op==SQLITE_DELETE; } @@ -238522,7 +239500,7 @@ static int sessionOneChangeToHash( if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, - pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange + pGrp->bPatch, pExist, op, bIndirect, aRec, nRec, &pChange ); } if( rc==SQLITE_OK && pChange ){ @@ -238531,6 +239509,47 @@ static int sessionOneChangeToHash( pTab->nEntry++; } + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeIterToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + const char *zTab = 0; + int nCol = 0; + int op = 0; + int bIndirect = 0; + int rc = SQLITE_OK; + SessionTable *pTab = 0; + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK ){ + rc = sessionOneChangeToHash( + pGrp, pTab, op, bIndirect, nCol, aRec, nRec, bRebase + ); + } + if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -238550,7 +239569,7 @@ static int sessionChangesetToHash( pIter->in.bNoDiscard = 1; while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ - rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + rc = sessionOneChangeIterToHash(pGrp, pIter, bRebase); if( rc!=SQLITE_OK ) break; } @@ -238640,6 +239659,33 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp){ return rc; } +/* +** Configure a changegroup object. +*/ +SQLITE_API int sqlite3changegroup_config( + sqlite3_changegroup *pGrp, + int op, + void *pArg +){ + int rc = SQLITE_OK; + + switch( op ){ + case SQLITE_CHANGEGROUP_CONFIG_PATCHSET: { + int arg = *(int*)pArg; + if( pGrp->pList==0 && arg>=0 ){ + pGrp->bPatch = (arg>0); + } + *(int*)pArg = pGrp->bPatch; + break; + } + default: + rc = SQLITE_MISUSE; + break; + } + + return rc; +} + /* ** Provide a database schema to the changegroup object. */ @@ -238698,7 +239744,7 @@ SQLITE_API int sqlite3changegroup_add_change( rc = SQLITE_ERROR; }else{ pIter->in.bNoDiscard = 1; - rc = sessionOneChangeToHash(pGrp, pIter, 0); + rc = sessionOneChangeIterToHash(pGrp, pIter, 0); } return rc; } @@ -238750,6 +239796,12 @@ SQLITE_API int sqlite3changegroup_output_strm( */ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ + int ii; + for(ii=0; iicd.nBufAlloc; ii++){ + sqlite3_free(pGrp->cd.aBuf[ii].aBuf); + } + sqlite3_free(pGrp->cd.record.aBuf); + sqlite3_free(pGrp->cd.aBuf); sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); sqlite3_free(pGrp->rec.aBuf); @@ -239180,6 +240232,328 @@ SQLITE_API int sqlite3session_config(int op, void *pArg){ return rc; } +/* +** Begin adding a change to a changegroup object. +*/ +SQLITE_API int sqlite3changegroup_change_begin( + sqlite3_changegroup *pGrp, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +){ + SessionTable *pTab = 0; + int rc = SQLITE_OK; + + if( pGrp->cd.pTab ){ + rc = SQLITE_MISUSE; + }else if( eOp!=SQLITE_INSERT && eOp!=SQLITE_UPDATE && eOp!=SQLITE_DELETE ){ + rc = SQLITE_ERROR; + }else{ + rc = sessionChangesetFindTable(pGrp, zTab, 0, &pTab); + } + if( rc==SQLITE_OK ){ + if( pTab==0 ){ + if( pzErr ){ + *pzErr = sqlite3_mprintf("no such table: %s", zTab); + } + rc = SQLITE_ERROR; + }else{ + int nReq = pTab->nCol * (eOp==SQLITE_UPDATE ? 2 : 1); + pGrp->cd.pTab = pTab; + pGrp->cd.eOp = eOp; + pGrp->cd.bIndirect = bIndirect; + + if( pGrp->cd.nBufAlloccd.aBuf, nReq * sizeof(SessionBuffer) + ); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&aBuf[pGrp->cd.nBufAlloc], 0, + sizeof(SessionBuffer) * (nReq - pGrp->cd.nBufAlloc) + ); + pGrp->cd.aBuf = aBuf; + pGrp->cd.nBufAlloc = nReq; + } + } + +#ifdef SQLITE_DEBUG + { + /* Assert that all column values are currently undefined */ + int ii; + for(ii=0; iicd.nBufAlloc; ii++){ + assert( pGrp->cd.aBuf[ii].nBuf==0 ); + } + } +#endif + } + } + + return rc; +} + +/* +** This function does processing common to the _change_int64(), _change_text() +** and other similar APIs. +*/ +static int checkChangeParams( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + sqlite3_int64 nReq, + SessionBuffer **ppBuf +){ + int rc = SQLITE_OK; + if( pGrp->cd.pTab==0 ){ + rc = SQLITE_MISUSE; + }else if( iCol<0 || iCol>=pGrp->cd.pTab->nCol ){ + rc = SQLITE_RANGE; + }else if( + (bNew && pGrp->cd.eOp==SQLITE_DELETE) + || (!bNew && pGrp->cd.eOp==SQLITE_INSERT) + ){ + rc = SQLITE_ERROR; + }else{ + SessionBuffer *pBuf = &pGrp->cd.aBuf[iCol]; + if( pGrp->cd.eOp==SQLITE_UPDATE && bNew ){ + pBuf += pGrp->cd.pTab->nCol; + } + pBuf->nBuf = 0; + sessionBufferGrow(pBuf, nReq, &rc); + pBuf->nBuf = nReq; + *ppBuf = pBuf; + } + return rc; +} + +/* +** Configure the change currently under construction with an integer value. +*/ +SQLITE_API int sqlite3changegroup_change_int64( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + sqlite3_int64 iVal +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 9, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_INTEGER; + sessionPutI64(&pBuf->aBuf[1], iVal); + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a null value. +*/ +SQLITE_API int sqlite3changegroup_change_null( + sqlite3_changegroup *pGrp, + int bNew, + int iCol +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 1, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_NULL; + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a real value. +*/ +SQLITE_API int sqlite3changegroup_change_double( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + double fVal +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 9, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_FLOAT; + sessionPutDouble(&pBuf->aBuf[1], fVal); + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a text value. +*/ +SQLITE_API int sqlite3changegroup_change_text( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + const char *pVal, + int nVal +){ + int nText = nVal>=0 ? nVal : strlen(pVal); + sqlite3_int64 nByte = 1 + sessionVarintLen(nText) + nText; + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, nByte, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_TEXT; + pBuf->nBuf = (1 + sessionVarintPut(&pBuf->aBuf[1], nText)); + memcpy(&pBuf->aBuf[pBuf->nBuf], pVal, nText); + pBuf->nBuf += nText; + + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a blob value. +*/ +SQLITE_API int sqlite3changegroup_change_blob( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + const void *pVal, + int nVal +){ + sqlite3_int64 nByte = 1 + sessionVarintLen(nVal) + nVal; + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, nByte, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_BLOB; + pBuf->nBuf = (1 + sessionVarintPut(&pBuf->aBuf[1], nVal)); + memcpy(&pBuf->aBuf[pBuf->nBuf], pVal, nVal); + pBuf->nBuf += nVal; + + return SQLITE_OK; +} + +/* +** Finish any change currently being constructed by the changegroup object. +*/ +SQLITE_API int sqlite3changegroup_change_finish( + sqlite3_changegroup *pGrp, + int bDiscard, + char **pzErr +){ + int rc = SQLITE_OK; + if( pGrp->cd.pTab ){ + SessionBuffer *aBuf = pGrp->cd.aBuf; + int ii; + + if( bDiscard==0 ){ + int nBuf = pGrp->cd.pTab->nCol; + u8 eUndef = SQLITE_NULL; + if( pGrp->cd.eOp==SQLITE_UPDATE ){ + for(ii=0; iicd.pTab->abPK[ii] ){ + if( aBuf[ii].nBuf<=1 ){ + *pzErr = sqlite3_mprintf( + "invalid change: %s value in PK of old.* record", + aBuf[ii].nBuf==1 ? "null" : "undefined" + ); + rc = SQLITE_ERROR; + break; + }else if( aBuf[ii + nBuf].nBuf>0 ){ + *pzErr = sqlite3_mprintf( + "invalid change: defined value in PK of new.* record" + ); + rc = SQLITE_ERROR; + break; + } + }else + if( pGrp->bPatch==0 && (aBuf[ii].nBuf>0)!=(aBuf[ii+nBuf].nBuf>0) ){ + *pzErr = sqlite3_mprintf( + "invalid change: column %d " + "- old.* value is %sdefined but new.* is %sdefined", + ii, aBuf[ii].nBuf ? "" : "un", aBuf[ii+nBuf].nBuf ? "" : "un" + ); + rc = SQLITE_ERROR; + break; + } + } + eUndef = 0x00; + if( pGrp->bPatch==0 ) nBuf = nBuf * 2; + }else{ + for(ii=0; iicd.pTab->abPK[ii]; + if( (pGrp->cd.eOp==SQLITE_INSERT || pGrp->bPatch==0 || isPK) + && aBuf[ii].nBuf==0 + ){ + *pzErr = sqlite3_mprintf( + "invalid change: column %d is undefined", ii + ); + rc = SQLITE_ERROR; + break; + } + if( aBuf[ii].nBuf==1 && isPK ){ + *pzErr = sqlite3_mprintf( + "invalid change: null value in PK" + ); + rc = SQLITE_ERROR; + break; + } + } + } + + pGrp->cd.record.nBuf = 0; + for(ii=0; iicd.aBuf[ii]; + if( pGrp->bPatch ){ + if( pGrp->cd.pTab->abPK[ii]==0 ){ + if( pGrp->cd.eOp==SQLITE_UPDATE ){ + p += pGrp->cd.pTab->nCol; + }else if( pGrp->cd.eOp==SQLITE_DELETE ){ + continue; + } + } + } + if( 0==sessionBufferGrow(&pGrp->cd.record, p->nBuf?p->nBuf:1, &rc) ){ + if( p->nBuf ){ + memcpy(&pGrp->cd.record.aBuf[pGrp->cd.record.nBuf],p->aBuf,p->nBuf); + pGrp->cd.record.nBuf += p->nBuf; + }else{ + pGrp->cd.record.aBuf[pGrp->cd.record.nBuf++] = eUndef; + } + } + } + if( rc==SQLITE_OK ){ + rc = sessionOneChangeToHash( + pGrp, pGrp->cd.pTab, + pGrp->cd.eOp, pGrp->cd.bIndirect, pGrp->cd.pTab->nCol, + pGrp->cd.record.aBuf, pGrp->cd.record.nBuf, 0 + ); + } + } + + /* Reset all aBuf[] entries to "undefined". */ + { + int nZero = pGrp->cd.pTab->nCol; + if( pGrp->cd.eOp==SQLITE_UPDATE ) nZero += nZero; + for(ii=0; iicd.aBuf[ii].nBuf = 0; + } + } + pGrp->cd.pTab = 0; + } + + return rc; +} + #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ /************** End of sqlite3session.c **************************************/ @@ -258777,7 +260151,7 @@ static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ if( sqlite3_libversion_number()>=3008002 ) #endif { - pIdxInfo->estimatedRows = nRow; + pIdxInfo->estimatedRows = MAX(1, nRow); } #endif } @@ -258846,19 +260220,30 @@ static int fts5UsePatternMatch( ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: ** -** * No other constraints: cost=1000.0 -** * One rowid range constraint: cost=750.0 -** * Both rowid range constraints: cost=500.0 -** * An == rowid constraint: cost=100.0 +** * No other constraints: cost=50000.0 +** * One rowid range constraint: cost=37500.0 +** * Both rowid range constraints: cost=30000.0 +** * An == rowid constraint: cost=25000.0 ** ** b) Otherwise, if there is no MATCH: ** -** * No other constraints: cost=1000000.0 -** * One rowid range constraint: cost=750000.0 -** * Both rowid range constraints: cost=250000.0 -** * An == rowid constraint: cost=10.0 +** * No other constraints: cost=3000000.0 +** * One rowid range constraints: cost=2250000.0 +** * Both rowid range constraint: cost=750000.0 +** * An == rowid constraint: cost=25.0 ** ** Costs are not modified by the ORDER BY clause. +** +** The ratios used in case (a) are based on informal results obtained from +** the tool/fts5cost.tcl script. The "MATCH and ==" combination has the +** cost set quite high because the query may be a prefix query. Unless +** there is a prefix index, prefix queries with rowid constraints are much +** more expensive than non-prefix queries with rowid constraints. +** +** The estimated rows returned is set to the cost/40. For simple queries, +** experimental results show that cost/4 might be about right. But for +** more complex queries that use multiple terms the number of rows might +** be far fewer than this. So we compromise and use cost/40. */ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Table *pTab = (Fts5Table*)pVTab; @@ -258984,21 +260369,35 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = nSeenMatch ? 1000.0 : 25.0; - fts5SetUniqueFlag(pInfo); + pInfo->estimatedCost = nSeenMatch ? 25000.0 : 25.0; fts5SetEstimatedRows(pInfo, 1); + fts5SetUniqueFlag(pInfo); }else{ - if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 5000.0 : 750000.0; - }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 7500.0 : 2250000.0; + i64 nEstRows; + if( nSeenMatch ){ + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = 50000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = 37500.0; + }else{ + pInfo->estimatedCost = 50000.0; + } + nEstRows = (i64)(pInfo->estimatedCost / 40.0); + for(i=1; iestimatedCost *= 2.5; + nEstRows = nEstRows / 2; + } }else{ - pInfo->estimatedCost = nSeenMatch ? 10000.0 : 3000000.0; - } - for(i=1; iestimatedCost *= 0.4; + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = 750000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = 2250000.0; + }else{ + pInfo->estimatedCost = 3000000.0; + } + nEstRows = (i64)(pInfo->estimatedCost / 4.0); } - fts5SetEstimatedRows(pInfo, (i64)(pInfo->estimatedCost / 4.0)); + fts5SetEstimatedRows(pInfo, nEstRows); } pInfo->idxNum = idxFlags; @@ -261859,7 +263258,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2026-03-06 16:01:44 557aeb43869d3585137b17690cb3b64f7de6921774daae9e56403c3717dceab6", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2026-05-05 10:34:17 c88b22011a54b4f6fbd149e9f8e4de77658ce58143a1af0e3785e4e6475127e9", -1, SQLITE_TRANSIENT); } /* diff --git a/cgosqlite/sqlite3.h b/cgosqlite/sqlite3.h index 3385675..8ee26c9 100644 --- a/cgosqlite/sqlite3.h +++ b/cgosqlite/sqlite3.h @@ -146,12 +146,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.52.0" -#define SQLITE_VERSION_NUMBER 3052000 -#define SQLITE_SOURCE_ID "2026-03-06 16:01:44 557aeb43869d3585137b17690cb3b64f7de6921774daae9e56403c3717dceab6" -#define SQLITE_SCM_BRANCH "trunk" -#define SQLITE_SCM_TAGS "release major-release version-3.52.0" -#define SQLITE_SCM_DATETIME "2026-03-06T16:01:44.367Z" +#define SQLITE_VERSION "3.53.1" +#define SQLITE_VERSION_NUMBER 3053001 +#define SQLITE_SOURCE_ID "2026-05-05 10:34:17 c88b22011a54b4f6fbd149e9f8e4de77658ce58143a1af0e3785e4e6475127e9" +#define SQLITE_SCM_BRANCH "branch-3.53" +#define SQLITE_SCM_TAGS "release version-3.53.1" +#define SQLITE_SCM_DATETIME "2026-05-05T10:34:17.344Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -578,7 +578,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal only */ /* ** CAPI3REF: Flags For File Open Operations @@ -1290,6 +1290,12 @@ struct sqlite3_io_methods { #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO +/* reserved file-control numbers: +** 101 +** 102 +** 103 +*/ + /* ** CAPI3REF: Mutex Handle @@ -1711,7 +1717,8 @@ SQLITE_API int sqlite3_os_end(void); ** are called "anytime configuration options". ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before ** [sqlite3_shutdown()] with a first argument that is not an anytime -** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** configuration option, then the sqlite3_config() call will +** return SQLITE_MISUSE. ** Note, however, that ^sqlite3_config() can be called as part of the ** implementation of an application-defined [sqlite3_os_init()]. ** @@ -2277,9 +2284,10 @@ struct sqlite3_mem_methods { ** is less than 8. The "sz" argument should be a multiple of 8 less than ** 65536. If "sz" does not meet this constraint, it is reduced in size until ** it does. -**

  • The third argument ("cnt") is the number of slots. Lookaside is disabled -** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so -** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +**

  • The third argument ("cnt") is the number of slots. +** Lookaside is disabled if "cnt"is less than 1. +* The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" ** parameter is usually chosen so that the product of "sz" and "cnt" is less ** than 1,000,000. ** @@ -2658,7 +2666,7 @@ struct sqlite3_mem_methods { ** This option takes two arguments which are an integer and a pointer ** to an integer. The first argument is a small integer, between 3 and 23, or ** zero. The FP_DIGITS setting is changed to that small integer, or left -** altered if the first argument is zero or out of range. The second argument +** unaltered if the first argument is zero or out of range. The second argument ** is a pointer to an integer. If the pointer is not NULL, then the value of ** the FP_DIGITS setting, after possibly being modified by the first ** arguments, is written into the integer to which the second argument points. @@ -2670,10 +2678,12 @@ struct sqlite3_mem_methods { ** **

    Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is an integer. -** The second argument is a pointer to an integer. If the first argument is 1, -** then the option becomes enabled. If the first integer argument is 0, then the -** option is disabled. If the first argument is -1, then the option setting +** The first argument (the third parameter to sqlite3_db_config()) is +** an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, +** then the option is disabled. +** If the first argument is -1, then the option setting ** is unchanged. The second argument, the pointer to an integer, may be NULL. ** If the second argument is not NULL, then a value of 0 or 1 is written into ** the integer to which the second argument points, depending on whether the @@ -4488,7 +4498,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. ** [sqlite3_prepare_v3()] has an extra -** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is some times +** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is sometimes ** needed for special purpose or to pass along security restrictions. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently @@ -5850,8 +5860,9 @@ SQLITE_API int sqlite3_create_window_function( ** of the strings must be aligned to a 2-byte boundary. ** ** [[SQLITE_UTF8_ZT]]

    SQLITE_UTF8_ZT
    This option can only be -** used to specify the text encoding to strings input to [sqlite3_result_text64()] -** and [sqlite3_bind_text64()]. It means that the input string (call it "z") +** used to specify the text encoding to strings input to +** [sqlite3_result_text64()] and [sqlite3_bind_text64()]. +** The SQLITE_UTF8_ZT encoding means that the input string (call it "z") ** is UTF-8 encoded and that it is zero-terminated. If the length parameter ** (call it "n") is non-negative, this encoding option means that the caller ** guarantees that z array contains at least n+1 bytes and that the z[n] @@ -6103,26 +6114,22 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** the SQL function that supplied the [sqlite3_value*] parameters. ** ** As long as the input parameter is correct, these routines can only -** fail if an out-of-memory error occurs during a format conversion. -** Only the following subset of interfaces are subject to out-of-memory -** errors: -** -**
      -**
    • sqlite3_value_blob() -**
    • sqlite3_value_text() -**
    • sqlite3_value_text16() -**
    • sqlite3_value_text16le() -**
    • sqlite3_value_text16be() -**
    • sqlite3_value_bytes() -**
    • sqlite3_value_bytes16() -**
    -** +** fail if an out-of-memory error occurs while trying to do a +** UTF8→UTF16 or UTF16→UTF8 conversion. ** If an out-of-memory error occurs, then the return value from these ** routines is the same as if the column had contained an SQL NULL value. -** Valid SQL NULL returns can be distinguished from out-of-memory errors -** by invoking the [sqlite3_errcode()] immediately after the suspect +** If the input sqlite3_value was not obtained from [sqlite3_value_dup()], +** then valid SQL NULL returns can also be distinguished from +** out-of-memory errors after extracting the value +** by invoking the [sqlite3_errcode()] immediately after the suspicious ** return value is obtained and before any ** other SQLite interface is called on the same [database connection]. +** If the input sqlite3_value was obtained from sqlite3_value_dup() then +** it is disconnected from the database connection and so sqlite3_errcode() +** will not work. +** In that case, the only way to distinguish an out-of-memory +** condition from a true SQL NULL is to invoke sqlite3_value_type() on the +** input to see if it is NULL prior to trying to extract the value. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -6149,7 +6156,8 @@ SQLITE_API int sqlite3_value_frombind(sqlite3_value*); ** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) ** returns something other than SQLITE_TEXT, then the return value from ** sqlite3_value_encoding(X) is meaningless. ^Calls to -** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], +** [sqlite3_value_text16be(X)], ** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or ** [sqlite3_value_bytes16(X)] might change the encoding of the value X and ** thus change the return from subsequent calls to sqlite3_value_encoding(X). @@ -6280,17 +6288,17 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** query execution, under some circumstances the associated auxiliary data ** might be preserved. An example of where this might be useful is in a ** regular-expression matching function. The compiled version of the regular -** expression can be stored as auxiliary data associated with the pattern string. -** Then as long as the pattern string remains the same, +** expression can be stored as auxiliary data associated with the pattern +** string. Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data -** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument -** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no auxiliary data -** associated with the function argument, the sqlite3_get_auxdata(C,N) interface -** returns a NULL pointer. +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary +** data associated by the sqlite3_set_auxdata(C,N,P,X) function with the +** Nth argument value to the application-defined function. ^N is zero +** for the left-most function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) +** interface returns a NULL pointer. ** ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the ** N-th argument of the application-defined function. ^Subsequent @@ -6495,7 +6503,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], ** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that ** the result text is both UTF-8 and zero-terminated. In other words, -** SQLITE_UTF8_ZT means that the Z array holds at least N+1 byes and that +** SQLITE_UTF8_ZT means that the Z array holds at least N+1 bytes and that ** the Z[N] is zero. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. @@ -7607,7 +7615,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** 
     **    int xEntryPoint(
 **      sqlite3 *db,
-**      const char **pzErrMsg,
+**      char **pzErrMsg,
 **      const struct sqlite3_api_routines *pThunk
 **    );
 **
    )^ @@ -8357,13 +8365,6 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor -** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex -** implementation is included with the library. In this case the -** application must supply a custom mutex implementation using the -** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function -** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() @@ -8718,6 +8719,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_ATOF 34 #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* @@ -8865,7 +8867,7 @@ SQLITE_API void sqlite3_str_free(sqlite3_str*); ** inside [sqlite3_str] object X back to zero bytes in length. ** ** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string -** under construction to be N bytes are less. This routine is a no-op if +** under construction to be N bytes or less. This routine is a no-op if ** N is negative or if the string is already N bytes or smaller in size. ** ** These methods do not return a result code. ^If an error occurs, that fact @@ -10409,7 +10411,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** sqlite3_vtab_distinct() return value ** Rows are returned in aOrderBy order -** Rows with the same value in all aOrderBy columns are adjacent +** Rows with the same value in all aOrderBy columns are +** adjacent ** Duplicates over all colUsed columns may be omitted ** 0yesyesno ** 1noyesno @@ -10418,8 +10421,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** ** ^For the purposes of comparing virtual table output values to see if the -** values are the same value for sorting purposes, two NULL values are considered -** to be the same. In other words, the comparison operator is "IS" +** values are the same value for sorting purposes, two NULL values are +** considered to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** ** If a virtual table implementation is unable to meet the requirements @@ -10729,7 +10732,7 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] and the -** [nexec and ncycle] columnes of the [bytecode virtual table]. +** [nexec and ncycle] columns of the [bytecode virtual table]. */ SQLITE_API int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -11273,29 +11276,30 @@ SQLITE_API int sqlite3_deserialize( ** ** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to ** parameter that is the first argument of the [carray() table-valued function]. -** The S parameter is a pointer to the [prepared statement] that uses the carray() -** functions. I is the parameter index to be bound. I must be the index of the -** parameter that is the first argument to the carray() table-valued function. -** P is a pointer to the array to be bound, and N is the number of elements in -** the array. The F argument is one of constants [SQLITE_CARRAY_INT32], -** [SQLITE_CARRAY_INT64], [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], +** The S parameter is a pointer to the [prepared statement] that uses the +** carray() functions. I is the parameter index to be bound. I must be the +** index of the parameter that is the first argument to the carray() +** table-valued function. P is a pointer to the array to be bound, and N +** is the number of elements in the array. The F argument is one of +** constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], ** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. ** ** If the X argument is not a NULL pointer or one of the special ** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke ** the function X with argument D when it is finished using the data in P. ** The call to X(D) is a destructor for the array P. The destructor X(D) -** is invoked even if the call to sqlite3_carray_bind() fails. If the X +** is invoked even if the call to sqlite3_carray_bind_v2() fails. If the X ** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes ** that the data static and the destructor is never invoked. If the X ** parameter is the special-case value [SQLITE_TRANSIENT], then ** sqlite3_carray_bind_v2() makes its own private copy of the data prior ** to returning and never invokes the destructor X. ** -** The sqlite3_carray_bind() function works the same as sqlite_carray_bind_v2() +** The sqlite3_carray_bind() function works the same as sqlite3_carray_bind_v2() ** with a D parameter set to P. In other words, ** sqlite3_carray_bind(S,I,P,N,F,X) is same as -** sqlite3_carray_bind(S,I,P,N,F,X,P). +** sqlite3_carray_bind_v2(S,I,P,N,F,X,P). */ SQLITE_API int sqlite3_carray_bind_v2( sqlite3_stmt *pStmt, /* Statement to be bound */ @@ -13336,6 +13340,232 @@ SQLITE_API int sqlite3session_config(int op, void *pArg); */ #define SQLITE_SESSION_CONFIG_STRMSIZE 1 +/* +** CAPI3REF: Configure a changegroup object +** +** Configure the changegroup object passed as the first argument. +** At present the only valid value for the second parameter is +** [SQLITE_CHANGEGROUP_CONFIG_PATCHSET]. +*/ +SQLITE_API int sqlite3changegroup_config(sqlite3_changegroup*, int, void *pArg); + +/* +** CAPI3REF: Options for sqlite3changegroup_config(). +** +** The following values may be passed as the 2nd parameter to +** sqlite3changegroup_config(). +** +**
    SQLITE_CHANGEGROUP_CONFIG_PATCHSET
    +** A changegroup object generates either a changeset or patchset. Usually, +** this is determined by whether the first call to sqlite3changegroup_add() +** is passed a changeset or a patchset. Or, if the first changes are added +** to the changegroup object using the sqlite3changegroup_change_xxx() +** APIs, then this option may be used to configure whether the changegroup +** object generates a changeset or patchset. +** +** When this option is invoked, parameter pArg must point to a value of +** type int. If the changegroup currently contains zero changes, and the +** value of the int variable is zero or greater than zero, then the +** changegroup is configured to generate a changeset or patchset, +** respectively. It is a no-op, not an error, if the changegroup is not +** configured because it has already started accumulating changes. +** +** Before returning, the int variable is set to 0 if the changegroup is +** configured to generate a changeset, or 1 if it is configured to generate +** a patchset. +*/ +#define SQLITE_CHANGEGROUP_CONFIG_PATCHSET 1 + + +/* +** CAPI3REF: Begin adding a change to a changegroup +** +** This API is used, in concert with other sqlite3changegroup_change_xxx() +** APIs, to add changes to a changegroup object one at a time. To add a +** single change, the caller must: +** +** 1. Invoke sqlite3changegroup_change_begin() to indicate the type of +** change (INSERT, UPDATE or DELETE), the affected table and whether +** or not the change should be marked as indirect. +** +** 2. Invoke sqlite3changegroup_change_int64() or one of the other four +** value functions - _null(), _double(), _text() or _blob() - one or +** more times to specify old.* and new.* values for the change being +** constructed. +** +** 3. Invoke sqlite3changegroup_change_finish() to either finish adding +** the change to the group, or to discard the change altogether. +** +** The first argument to this function must be a pointer to the existing +** changegroup object that the change will be added to. The second argument +** must be SQLITE_INSERT, SQLITE_UPDATE or SQLITE_DELETE. The third is the +** name of the table that the change affects, and the fourth is a boolean +** flag specifying whether the change should be marked as "indirect" (if +** bIndirect is non-zero) or not indirect (if bIndirect is zero). +** +** Following a successful call to this function, this function may not be +** called again on the same changegroup object until after +** sqlite3changegroup_change_finish() has been called. Doing so is an +** SQLITE_MISUSE error. +** +** The changegroup object passed as the first argument must be already +** configured with schema data for the specified table. It may be configured +** either by calling sqlite3changegroup_schema() with a database that contains +** the table, or sqlite3changegroup_add() with a changeset that contains the +** table. If the changegroup object has not been configured with a schema for +** the specified table when this function is called, SQLITE_ERROR is returned. +** +** If successful, SQLITE_OK is returned. Otherwise, if an error occurs, an +** SQLite error code is returned. In this case, if argument pzErr is non-NULL, +** then (*pzErr) may be set to point to a buffer containing a utf-8 formated, +** nul-terminated, English language error message. It is the responsibility +** of the caller to eventually free this buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_begin( + sqlite3_changegroup*, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +); + +/* +** CAPI3REF: Add a 64-bit integer to a changegroup +** +** This function may only be called between a successful call to +** sqlite3changegroup_change_begin() and its matching +** sqlite3changegroup_change_finish() call. If it is called at any +** other time, it is an SQLITE_MISUSE error. Calling this function +** specifies a 64-bit integer value to be used in the change currently being +** added to the changegroup object passed as the first argument. +** +** The second parameter, bNew, specifies whether the value is to be part of +** the new.* (if bNew is non-zero) or old.* (if bNew is zero) record of +** the change under construction. If this does not match the type of change +** specified by the preceding call to sqlite3changegroup_change_begin() (i.e. +** an old.* value for an SQLITE_INSERT change, or a new.* value for an +** SQLITE_DELETE), then SQLITE_ERROR is returned. +** +** The third parameter specifies the column of the old.* or new.* record that +** the value will be a part of. If the specified table has an explicit primary +** key, then this is the index of the table column, numbered from 0 in the order +** specified within the CREATE TABLE statement. Or, if the table uses an +** implicit rowid key, then the column 0 is the rowid and the explicit columns +** are numbered starting from 1. If the iCol parameter is less than 0 or greater +** than the index of the last column in the table, SQLITE_RANGE is returned. +** +** The fourth parameter is the integer value to use as part of the old.* or +** new.* record. +** +** If this call is successful, SQLITE_OK is returned. Otherwise, if an +** error occurs, an SQLite error code is returned. +*/ +SQLITE_API int sqlite3changegroup_change_int64( + sqlite3_changegroup*, + int bNew, + int iCol, + sqlite3_int64 iVal +); + +/* +** CAPI3REF: Add a NULL to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently under construction with a NULL value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_null(sqlite3_changegroup*, int, int); + +/* +** CAPI3REF: Add an double to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently being constructed with a real value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_double(sqlite3_changegroup*, int, int, double); + +/* +** CAPI3REF: Add a text value to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a text value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the text encoded using +** utf-8. Parameter nVal may either be the size of the text value in bytes, or +** else a negative value, in which case the buffer pVal points to is assumed to +** be nul-terminated. +*/ +SQLITE_API int sqlite3changegroup_change_text( + sqlite3_changegroup*, int, int, const char *pVal, int nVal +); + +/* +** CAPI3REF: Add a blob to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a blob value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the blob. Parameter +** nVal is the size of the blob in bytes. +*/ +SQLITE_API int sqlite3changegroup_change_blob( + sqlite3_changegroup*, int, int, const void *pVal, int nVal +); + +/* +** CAPI3REF: Finish adding one-at-at-time changes to a changegroup +** +** This function may only be called following a successful call to +** sqlite3changegroup_change_begin(). Otherwise, it is an SQLITE_MISUSE error. +** +** If parameter bDiscard is non-zero, then the current change is simply +** discarded. In this case this function is always successful and SQLITE_OK +** returned. +** +** If parameter bDiscard is zero, then an attempt is made to add the current +** change to the changegroup. Assuming the changegroup is configured to +** produce a changeset (not a patchset), this requires that: +** +** * If the change is an INSERT or DELETE, then a value must be specified +** for all columns of the new.* or old.* record, respectively. +** +** * If the change is an UPDATE record, then values must be provided for +** the PRIMARY KEY columns of the old.* record, but must not be provided +** for PRIMARY KEY columns of the new.* record. +** +** * If the change is an UPDATE record, then for each non-PRIMARY KEY +** column in the old.* record for which a value has been provided, a +** value must also be provided for the same column in the new.* record. +** Similarly, for each non-PK column in the old.* record for which +** a value is not provided, a value must not be provided for the same +** column in the new.* record. +** +** * All values specified for PRIMARY KEY columns must be non-NULL. +** +** Otherwise, it is an error. +** +** If the changegroup already contains a change for the same row (identified +** by PRIMARY KEY columns), then the current change is combined with the +** existing change in the same way as for sqlite3changegroup_add(). +** +** For a patchset, all of the above rules apply except that it doesn't matter +** whether or not values are provided for the non-PK old.* record columns +** for an UPDATE or DELETE change. This means that code used to produce +** a changeset using the sqlite3changegroup_change_xxx() APIs may also +** be used to produce patchsets. +** +** If the call is successful, SQLITE_OK is returned. Otherwise, if an error +** occurs, an SQLite error code is returned. If an error is returned and +** parameter pzErr is not NULL, then (*pzErr) may be set to point to a buffer +** containing a nul-terminated, utf-8 encoded, English language error message. +** It is the responsibility of the caller to eventually free any such error +** message buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_finish( + sqlite3_changegroup*, + int bDiscard, + char **pzErr +); + /* ** Make sure we can call this stuff from C++. */ diff --git a/update-sqlite.sh b/update-sqlite.sh index 4e24e67..e22a3a9 100755 --- a/update-sqlite.sh +++ b/update-sqlite.sh @@ -46,3 +46,5 @@ printf "#ifndef SQLITE_TRUNK\n\n" > sqlite3.c cat sqlite3.c_partial >> sqlite3.c printf "\n\n#endif\n" >> sqlite3.c rm sqlite3.c_partial + +echo "Next step: update the expected version numbers in release_test.go"